From eb1181594417dafad0f75808ead71f6d5170c1ea Mon Sep 17 00:00:00 2001 From: David Howells Date: Mon, 16 Dec 2024 20:40:53 +0000 Subject: netfs: Use a folio_queue allocation and free functions Provide and use folio_queue allocation and free functions to combine the allocation, initialisation and stat (un)accounting steps that are repeated in several places. Signed-off-by: David Howells Link: https://lore.kernel.org/r/20241216204124.3752367-4-dhowells@redhat.com cc: Jeff Layton cc: netfs@lists.linux.dev cc: linux-fsdevel@vger.kernel.org Signed-off-by: Christian Brauner --- fs/netfs/misc.c | 38 ++++++++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) (limited to 'fs/netfs/misc.c') diff --git a/fs/netfs/misc.c b/fs/netfs/misc.c index 78fe5796b2b2f..6cd7e1ee7a145 100644 --- a/fs/netfs/misc.c +++ b/fs/netfs/misc.c @@ -8,6 +8,38 @@ #include #include "internal.h" +/** + * netfs_folioq_alloc - Allocate a folio_queue struct + * @gfp: Allocation constraints + * + * Allocate, initialise and account the folio_queue struct. + */ +struct folio_queue *netfs_folioq_alloc(gfp_t gfp) +{ + struct folio_queue *fq; + + fq = kmalloc(sizeof(*fq), gfp); + if (fq) { + netfs_stat(&netfs_n_folioq); + folioq_init(fq); + } + return fq; +} +EXPORT_SYMBOL(netfs_folioq_alloc); + +/** + * netfs_folioq_free - Free a folio_queue struct + * @folioq: The object to free + * + * Free and unaccount the folio_queue struct. + */ +void netfs_folioq_free(struct folio_queue *folioq) +{ + netfs_stat_d(&netfs_n_folioq); + kfree(folioq); +} +EXPORT_SYMBOL(netfs_folioq_free); + /* * Make sure there's space in the rolling queue. */ @@ -87,8 +119,7 @@ struct folio_queue *netfs_delete_buffer_head(struct netfs_io_request *wreq) if (next) next->prev = NULL; - netfs_stat_d(&netfs_n_folioq); - kfree(head); + netfs_folioq_free(head); wreq->buffer = next; return next; } @@ -111,8 +142,7 @@ void netfs_clear_buffer(struct netfs_io_request *rreq) folio_put(folio); } } - netfs_stat_d(&netfs_n_folioq); - kfree(p); + netfs_folioq_free(p); } } -- cgit v1.2.3 From aabcabf2746062253565b33aa3f8d25999a5ac01 Mon Sep 17 00:00:00 2001 From: David Howells Date: Mon, 16 Dec 2024 20:40:54 +0000 Subject: netfs: Add a tracepoint to log the lifespan of folio_queue structs Add a tracepoint to log the lifespan of folio_queue structs. For tracing illustrative purposes, folio_queues are tagged with the debug ID of whatever they're related to (typically a netfs_io_request) and a debug ID of their own. Signed-off-by: David Howells Link: https://lore.kernel.org/r/20241216204124.3752367-5-dhowells@redhat.com cc: Jeff Layton cc: netfs@lists.linux.dev cc: linux-fsdevel@vger.kernel.org Signed-off-by: Christian Brauner --- fs/netfs/buffered_read.c | 10 +++++++--- fs/netfs/internal.h | 3 ++- fs/netfs/misc.c | 31 ++++++++++++++++++++----------- fs/netfs/read_collect.c | 8 ++++++-- fs/netfs/write_issue.c | 2 +- fs/smb/client/smb2ops.c | 2 +- include/linux/folio_queue.h | 12 +++++++++--- include/linux/netfs.h | 6 ++++-- include/trace/events/netfs.h | 41 +++++++++++++++++++++++++++++++++++++++-- lib/kunit_iov_iter.c | 4 ++-- 10 files changed, 91 insertions(+), 28 deletions(-) (limited to 'fs/netfs/misc.c') diff --git a/fs/netfs/buffered_read.c b/fs/netfs/buffered_read.c index 78b04763bed60..7ec04d5628d80 100644 --- a/fs/netfs/buffered_read.c +++ b/fs/netfs/buffered_read.c @@ -131,7 +131,8 @@ static ssize_t netfs_prepare_read_iterator(struct netfs_io_subrequest *subreq) struct folio_queue *tail = rreq->buffer_tail, *new; size_t added; - new = netfs_folioq_alloc(GFP_NOFS); + new = netfs_folioq_alloc(rreq->debug_id, GFP_NOFS, + netfs_trace_folioq_alloc_read_prep); if (!new) return -ENOMEM; new->prev = tail; @@ -361,9 +362,11 @@ static int netfs_prime_buffer(struct netfs_io_request *rreq) struct folio_batch put_batch; size_t added; - folioq = netfs_folioq_alloc(GFP_KERNEL); + folioq = netfs_folioq_alloc(rreq->debug_id, GFP_KERNEL, + netfs_trace_folioq_alloc_read_prime); if (!folioq) return -ENOMEM; + rreq->buffer = folioq; rreq->buffer_tail = folioq; rreq->submitted = rreq->start; @@ -436,7 +439,8 @@ static int netfs_create_singular_buffer(struct netfs_io_request *rreq, struct fo { struct folio_queue *folioq; - folioq = netfs_folioq_alloc(GFP_KERNEL); + folioq = netfs_folioq_alloc(rreq->debug_id, GFP_KERNEL, + netfs_trace_folioq_alloc_read_sing); if (!folioq) return -ENOMEM; diff --git a/fs/netfs/internal.h b/fs/netfs/internal.h index c562aec3b483f..01b013f558f7f 100644 --- a/fs/netfs/internal.h +++ b/fs/netfs/internal.h @@ -58,7 +58,8 @@ static inline void netfs_proc_del_rreq(struct netfs_io_request *rreq) {} /* * misc.c */ -struct folio_queue *netfs_buffer_make_space(struct netfs_io_request *rreq); +struct folio_queue *netfs_buffer_make_space(struct netfs_io_request *rreq, + enum netfs_folioq_trace trace); int netfs_buffer_append_folio(struct netfs_io_request *rreq, struct folio *folio, bool needs_put); struct folio_queue *netfs_delete_buffer_head(struct netfs_io_request *wreq); diff --git a/fs/netfs/misc.c b/fs/netfs/misc.c index 6cd7e1ee7a145..afe032551de50 100644 --- a/fs/netfs/misc.c +++ b/fs/netfs/misc.c @@ -10,18 +10,25 @@ /** * netfs_folioq_alloc - Allocate a folio_queue struct + * @rreq_id: Associated debugging ID for tracing purposes * @gfp: Allocation constraints + * @trace: Trace tag to indicate the purpose of the allocation * - * Allocate, initialise and account the folio_queue struct. + * Allocate, initialise and account the folio_queue struct and log a trace line + * to mark the allocation. */ -struct folio_queue *netfs_folioq_alloc(gfp_t gfp) +struct folio_queue *netfs_folioq_alloc(unsigned int rreq_id, gfp_t gfp, + unsigned int /*enum netfs_folioq_trace*/ trace) { + static atomic_t debug_ids; struct folio_queue *fq; fq = kmalloc(sizeof(*fq), gfp); if (fq) { netfs_stat(&netfs_n_folioq); - folioq_init(fq); + folioq_init(fq, rreq_id); + fq->debug_id = atomic_inc_return(&debug_ids); + trace_netfs_folioq(fq, trace); } return fq; } @@ -30,11 +37,14 @@ EXPORT_SYMBOL(netfs_folioq_alloc); /** * netfs_folioq_free - Free a folio_queue struct * @folioq: The object to free + * @trace: Trace tag to indicate which free * * Free and unaccount the folio_queue struct. */ -void netfs_folioq_free(struct folio_queue *folioq) +void netfs_folioq_free(struct folio_queue *folioq, + unsigned int /*enum netfs_trace_folioq*/ trace) { + trace_netfs_folioq(folioq, trace); netfs_stat_d(&netfs_n_folioq); kfree(folioq); } @@ -43,7 +53,8 @@ EXPORT_SYMBOL(netfs_folioq_free); /* * Make sure there's space in the rolling queue. */ -struct folio_queue *netfs_buffer_make_space(struct netfs_io_request *rreq) +struct folio_queue *netfs_buffer_make_space(struct netfs_io_request *rreq, + enum netfs_folioq_trace trace) { struct folio_queue *tail = rreq->buffer_tail, *prev; unsigned int prev_nr_slots = 0; @@ -59,11 +70,9 @@ struct folio_queue *netfs_buffer_make_space(struct netfs_io_request *rreq) prev_nr_slots = folioq_nr_slots(tail); } - tail = kmalloc(sizeof(*tail), GFP_NOFS); + tail = netfs_folioq_alloc(rreq->debug_id, GFP_NOFS, trace); if (!tail) return ERR_PTR(-ENOMEM); - netfs_stat(&netfs_n_folioq); - folioq_init(tail); tail->prev = prev; if (prev) /* [!] NOTE: After we set prev->next, the consumer is entirely @@ -98,7 +107,7 @@ int netfs_buffer_append_folio(struct netfs_io_request *rreq, struct folio *folio struct folio_queue *tail; unsigned int slot, order = folio_order(folio); - tail = netfs_buffer_make_space(rreq); + tail = netfs_buffer_make_space(rreq, netfs_trace_folioq_alloc_append_folio); if (IS_ERR(tail)) return PTR_ERR(tail); @@ -119,7 +128,7 @@ struct folio_queue *netfs_delete_buffer_head(struct netfs_io_request *wreq) if (next) next->prev = NULL; - netfs_folioq_free(head); + netfs_folioq_free(head, netfs_trace_folioq_delete); wreq->buffer = next; return next; } @@ -142,7 +151,7 @@ void netfs_clear_buffer(struct netfs_io_request *rreq) folio_put(folio); } } - netfs_folioq_free(p); + netfs_folioq_free(p, netfs_trace_folioq_clear); } } diff --git a/fs/netfs/read_collect.c b/fs/netfs/read_collect.c index e8624f5c7fcc6..f7a5cb29dd126 100644 --- a/fs/netfs/read_collect.c +++ b/fs/netfs/read_collect.c @@ -103,6 +103,7 @@ static bool netfs_consume_read_data(struct netfs_io_subrequest *subreq, bool was subreq->transferred, subreq->len)) subreq->transferred = subreq->len; + trace_netfs_folioq(folioq, netfs_trace_folioq_read_progress); next_folio: fsize = PAGE_SIZE << subreq->curr_folio_order; fpos = round_down(subreq->start + subreq->consumed, fsize); @@ -119,9 +120,11 @@ next_folio: if (folioq) { struct folio *folio = folioq_folio(folioq, slot); - pr_err("folioq: orders=%02x%02x%02x%02x\n", + pr_err("folioq: fq=%x orders=%02x%02x%02x%02x %px\n", + folioq->debug_id, folioq->orders[0], folioq->orders[1], - folioq->orders[2], folioq->orders[3]); + folioq->orders[2], folioq->orders[3], + folioq); if (folio) pr_err("folio: %llx-%llx ix=%llx o=%u qo=%u\n", fpos, fend - 1, folio_pos(folio), folio_order(folio), @@ -222,6 +225,7 @@ donation_changed: slot = 0; folioq = folioq->next; subreq->curr_folioq = folioq; + trace_netfs_folioq(folioq, netfs_trace_folioq_read_progress); } subreq->curr_folioq_slot = slot; if (folioq && folioq_folio(folioq, slot)) diff --git a/fs/netfs/write_issue.c b/fs/netfs/write_issue.c index ff0e82505a0bd..87e5cf4a09576 100644 --- a/fs/netfs/write_issue.c +++ b/fs/netfs/write_issue.c @@ -161,7 +161,7 @@ static void netfs_prepare_write(struct netfs_io_request *wreq, */ if (iov_iter_is_folioq(wreq_iter) && wreq_iter->folioq_slot >= folioq_nr_slots(wreq_iter->folioq)) { - netfs_buffer_make_space(wreq); + netfs_buffer_make_space(wreq, netfs_trace_folioq_prep_write); } subreq = netfs_alloc_subrequest(wreq); diff --git a/fs/smb/client/smb2ops.c b/fs/smb/client/smb2ops.c index 87cb1872db28b..7121d9e0f4046 100644 --- a/fs/smb/client/smb2ops.c +++ b/fs/smb/client/smb2ops.c @@ -4388,7 +4388,7 @@ static struct folio_queue *cifs_alloc_folioq_buffer(ssize_t size) p = kmalloc(sizeof(*p), GFP_NOFS); if (!p) goto nomem; - folioq_init(p); + folioq_init(p, 0); if (tail) { tail->next = p; p->prev = tail; diff --git a/include/linux/folio_queue.h b/include/linux/folio_queue.h index 3abe614ef5f06..4d3f8074c137a 100644 --- a/include/linux/folio_queue.h +++ b/include/linux/folio_queue.h @@ -37,16 +37,20 @@ struct folio_queue { #if PAGEVEC_SIZE > BITS_PER_LONG #error marks is not big enough #endif + unsigned int rreq_id; + unsigned int debug_id; }; /** * folioq_init - Initialise a folio queue segment * @folioq: The segment to initialise + * @rreq_id: The request identifier to use in tracelines. * - * Initialise a folio queue segment. Note that the folio pointers are - * left uninitialised. + * Initialise a folio queue segment and set an identifier to be used in traces. + * + * Note that the folio pointers are left uninitialised. */ -static inline void folioq_init(struct folio_queue *folioq) +static inline void folioq_init(struct folio_queue *folioq, unsigned int rreq_id) { folio_batch_init(&folioq->vec); folioq->next = NULL; @@ -54,6 +58,8 @@ static inline void folioq_init(struct folio_queue *folioq) folioq->marks = 0; folioq->marks2 = 0; folioq->marks3 = 0; + folioq->rreq_id = rreq_id; + folioq->debug_id = 0; } /** diff --git a/include/linux/netfs.h b/include/linux/netfs.h index c69e0f02c30fb..5b2f427f8e3eb 100644 --- a/include/linux/netfs.h +++ b/include/linux/netfs.h @@ -455,8 +455,10 @@ int netfs_start_io_direct(struct inode *inode); void netfs_end_io_direct(struct inode *inode); /* Miscellaneous APIs. */ -struct folio_queue *netfs_folioq_alloc(gfp_t gfp); -void netfs_folioq_free(struct folio_queue *folioq); +struct folio_queue *netfs_folioq_alloc(unsigned int rreq_id, gfp_t gfp, + unsigned int trace /*enum netfs_folioq_trace*/); +void netfs_folioq_free(struct folio_queue *folioq, + unsigned int trace /*enum netfs_trace_folioq*/); /** * netfs_inode - Get the netfs inode context from the inode diff --git a/include/trace/events/netfs.h b/include/trace/events/netfs.h index c3c309f8fbe13..50aa6745df95f 100644 --- a/include/trace/events/netfs.h +++ b/include/trace/events/netfs.h @@ -191,6 +191,16 @@ EM(netfs_trace_donate_to_next, "to-next") \ E_(netfs_trace_donate_to_deferred_next, "defer-next") +#define netfs_folioq_traces \ + EM(netfs_trace_folioq_alloc_append_folio, "alloc-apf") \ + EM(netfs_trace_folioq_alloc_read_prep, "alloc-r-prep") \ + EM(netfs_trace_folioq_alloc_read_prime, "alloc-r-prime") \ + EM(netfs_trace_folioq_alloc_read_sing, "alloc-r-sing") \ + EM(netfs_trace_folioq_clear, "clear") \ + EM(netfs_trace_folioq_delete, "delete") \ + EM(netfs_trace_folioq_prep_write, "prep-wr") \ + E_(netfs_trace_folioq_read_progress, "r-progress") + #ifndef __NETFS_DECLARE_TRACE_ENUMS_ONCE_ONLY #define __NETFS_DECLARE_TRACE_ENUMS_ONCE_ONLY @@ -209,6 +219,7 @@ enum netfs_sreq_ref_trace { netfs_sreq_ref_traces } __mode(byte); enum netfs_folio_trace { netfs_folio_traces } __mode(byte); enum netfs_collect_contig_trace { netfs_collect_contig_traces } __mode(byte); enum netfs_donate_trace { netfs_donate_traces } __mode(byte); +enum netfs_folioq_trace { netfs_folioq_traces } __mode(byte); #endif @@ -232,6 +243,7 @@ netfs_sreq_ref_traces; netfs_folio_traces; netfs_collect_contig_traces; netfs_donate_traces; +netfs_folioq_traces; /* * Now redefine the EM() and E_() macros to map the enums to the strings that @@ -317,6 +329,7 @@ TRACE_EVENT(netfs_sreq, __field(unsigned short, flags) __field(enum netfs_io_source, source) __field(enum netfs_sreq_trace, what) + __field(u8, slot) __field(size_t, len) __field(size_t, transferred) __field(loff_t, start) @@ -332,15 +345,16 @@ TRACE_EVENT(netfs_sreq, __entry->len = sreq->len; __entry->transferred = sreq->transferred; __entry->start = sreq->start; + __entry->slot = sreq->curr_folioq_slot; ), - TP_printk("R=%08x[%x] %s %s f=%02x s=%llx %zx/%zx e=%d", + TP_printk("R=%08x[%x] %s %s f=%02x s=%llx %zx/%zx s=%u e=%d", __entry->rreq, __entry->index, __print_symbolic(__entry->source, netfs_sreq_sources), __print_symbolic(__entry->what, netfs_sreq_traces), __entry->flags, __entry->start, __entry->transferred, __entry->len, - __entry->error) + __entry->slot, __entry->error) ); TRACE_EVENT(netfs_failure, @@ -745,6 +759,29 @@ TRACE_EVENT(netfs_donate, __entry->amount) ); +TRACE_EVENT(netfs_folioq, + TP_PROTO(const struct folio_queue *fq, + enum netfs_folioq_trace trace), + + TP_ARGS(fq, trace), + + TP_STRUCT__entry( + __field(unsigned int, rreq) + __field(unsigned int, id) + __field(enum netfs_folioq_trace, trace) + ), + + TP_fast_assign( + __entry->rreq = fq ? fq->rreq_id : 0; + __entry->id = fq ? fq->debug_id : 0; + __entry->trace = trace; + ), + + TP_printk("R=%08x fq=%x %s", + __entry->rreq, __entry->id, + __print_symbolic(__entry->trace, netfs_folioq_traces)) + ); + #undef EM #undef E_ #endif /* _TRACE_NETFS_H */ diff --git a/lib/kunit_iov_iter.c b/lib/kunit_iov_iter.c index 13e15687675a8..10a560feb66e2 100644 --- a/lib/kunit_iov_iter.c +++ b/lib/kunit_iov_iter.c @@ -392,7 +392,7 @@ static void __init iov_kunit_load_folioq(struct kunit *test, if (folioq_full(p)) { p->next = kzalloc(sizeof(struct folio_queue), GFP_KERNEL); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, p->next); - folioq_init(p->next); + folioq_init(p->next, 0); p->next->prev = p; p = p->next; } @@ -409,7 +409,7 @@ static struct folio_queue *iov_kunit_create_folioq(struct kunit *test) folioq = kzalloc(sizeof(struct folio_queue), GFP_KERNEL); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folioq); kunit_add_action_or_reset(test, iov_kunit_destroy_folioq, folioq); - folioq_init(folioq); + folioq_init(folioq, 0); return folioq; } -- cgit v1.2.3 From 06fa229ceb36898e68022b5654c017d2c6582d7d Mon Sep 17 00:00:00 2001 From: David Howells Date: Mon, 16 Dec 2024 20:40:55 +0000 Subject: netfs: Abstract out a rolling folio buffer implementation A rolling buffer is a series of folios held in a list of folio_queues. New folios and folio_queue structs may be inserted at the head simultaneously with spent ones being removed from the tail without the need for locking. The rolling buffer includes an iov_iter and it has to be careful managing this as the list of folio_queues is extended such that an oops doesn't incurred because the iterator was pointing to the end of a folio_queue segment that got appended to and then removed. We need to use the mechanism twice, once for read and once for write, and, in future patches, we will use a second rolling buffer to handle bounce buffering for content encryption. Signed-off-by: David Howells Link: https://lore.kernel.org/r/20241216204124.3752367-6-dhowells@redhat.com cc: Jeff Layton cc: netfs@lists.linux.dev cc: linux-fsdevel@vger.kernel.org Signed-off-by: Christian Brauner --- fs/netfs/Makefile | 1 + fs/netfs/buffered_read.c | 119 +++++----------------- fs/netfs/direct_read.c | 14 +-- fs/netfs/direct_write.c | 10 +- fs/netfs/internal.h | 4 - fs/netfs/misc.c | 147 --------------------------- fs/netfs/objects.c | 2 +- fs/netfs/read_pgpriv2.c | 32 +++--- fs/netfs/read_retry.c | 2 +- fs/netfs/rolling_buffer.c | 226 +++++++++++++++++++++++++++++++++++++++++ fs/netfs/write_collect.c | 19 ++-- fs/netfs/write_issue.c | 26 +++-- include/linux/netfs.h | 10 +- include/linux/rolling_buffer.h | 61 +++++++++++ include/trace/events/netfs.h | 2 + 15 files changed, 375 insertions(+), 300 deletions(-) create mode 100644 fs/netfs/rolling_buffer.c create mode 100644 include/linux/rolling_buffer.h (limited to 'fs/netfs/misc.c') diff --git a/fs/netfs/Makefile b/fs/netfs/Makefile index d08b0bfb67569..7492c4aa331e2 100644 --- a/fs/netfs/Makefile +++ b/fs/netfs/Makefile @@ -13,6 +13,7 @@ netfs-y := \ read_collect.o \ read_pgpriv2.o \ read_retry.o \ + rolling_buffer.o \ write_collect.o \ write_issue.o diff --git a/fs/netfs/buffered_read.c b/fs/netfs/buffered_read.c index 7ec04d5628d80..db874fea87946 100644 --- a/fs/netfs/buffered_read.c +++ b/fs/netfs/buffered_read.c @@ -63,37 +63,6 @@ static int netfs_begin_cache_read(struct netfs_io_request *rreq, struct netfs_in return fscache_begin_read_operation(&rreq->cache_resources, netfs_i_cookie(ctx)); } -/* - * Decant the list of folios to read into a rolling buffer. - */ -static size_t netfs_load_buffer_from_ra(struct netfs_io_request *rreq, - struct folio_queue *folioq, - struct folio_batch *put_batch) -{ - unsigned int order, nr; - size_t size = 0; - - nr = __readahead_batch(rreq->ractl, (struct page **)folioq->vec.folios, - ARRAY_SIZE(folioq->vec.folios)); - folioq->vec.nr = nr; - for (int i = 0; i < nr; i++) { - struct folio *folio = folioq_folio(folioq, i); - - trace_netfs_folio(folio, netfs_folio_trace_read); - order = folio_order(folio); - folioq->orders[i] = order; - size += PAGE_SIZE << order; - - if (!folio_batch_add(put_batch, folio)) - folio_batch_release(put_batch); - } - - for (int i = nr; i < folioq_nr_slots(folioq); i++) - folioq_clear(folioq, i); - - return size; -} - /* * netfs_prepare_read_iterator - Prepare the subreq iterator for I/O * @subreq: The subrequest to be set up @@ -128,18 +97,12 @@ static ssize_t netfs_prepare_read_iterator(struct netfs_io_subrequest *subreq) folio_batch_init(&put_batch); while (rreq->submitted < subreq->start + rsize) { - struct folio_queue *tail = rreq->buffer_tail, *new; - size_t added; - - new = netfs_folioq_alloc(rreq->debug_id, GFP_NOFS, - netfs_trace_folioq_alloc_read_prep); - if (!new) - return -ENOMEM; - new->prev = tail; - tail->next = new; - rreq->buffer_tail = new; - added = netfs_load_buffer_from_ra(rreq, new, &put_batch); - rreq->iter.count += added; + ssize_t added; + + added = rolling_buffer_load_from_ra(&rreq->buffer, rreq->ractl, + &put_batch); + if (added < 0) + return added; rreq->submitted += added; } folio_batch_release(&put_batch); @@ -147,7 +110,7 @@ static ssize_t netfs_prepare_read_iterator(struct netfs_io_subrequest *subreq) subreq->len = rsize; if (unlikely(rreq->io_streams[0].sreq_max_segs)) { - size_t limit = netfs_limit_iter(&rreq->iter, 0, rsize, + size_t limit = netfs_limit_iter(&rreq->buffer.iter, 0, rsize, rreq->io_streams[0].sreq_max_segs); if (limit < rsize) { @@ -156,20 +119,16 @@ static ssize_t netfs_prepare_read_iterator(struct netfs_io_subrequest *subreq) } } - subreq->io_iter = rreq->iter; + subreq->io_iter = rreq->buffer.iter; if (iov_iter_is_folioq(&subreq->io_iter)) { - if (subreq->io_iter.folioq_slot >= folioq_nr_slots(subreq->io_iter.folioq)) { - subreq->io_iter.folioq = subreq->io_iter.folioq->next; - subreq->io_iter.folioq_slot = 0; - } subreq->curr_folioq = (struct folio_queue *)subreq->io_iter.folioq; subreq->curr_folioq_slot = subreq->io_iter.folioq_slot; subreq->curr_folio_order = subreq->curr_folioq->orders[subreq->curr_folioq_slot]; } iov_iter_truncate(&subreq->io_iter, subreq->len); - iov_iter_advance(&rreq->iter, subreq->len); + rolling_buffer_advance(&rreq->buffer, subreq->len); return subreq->len; } @@ -352,34 +311,6 @@ static int netfs_wait_for_read(struct netfs_io_request *rreq) return ret; } -/* - * Set up the initial folioq of buffer folios in the rolling buffer and set the - * iterator to refer to it. - */ -static int netfs_prime_buffer(struct netfs_io_request *rreq) -{ - struct folio_queue *folioq; - struct folio_batch put_batch; - size_t added; - - folioq = netfs_folioq_alloc(rreq->debug_id, GFP_KERNEL, - netfs_trace_folioq_alloc_read_prime); - if (!folioq) - return -ENOMEM; - - rreq->buffer = folioq; - rreq->buffer_tail = folioq; - rreq->submitted = rreq->start; - iov_iter_folio_queue(&rreq->iter, ITER_DEST, folioq, 0, 0, 0); - - folio_batch_init(&put_batch); - added = netfs_load_buffer_from_ra(rreq, folioq, &put_batch); - folio_batch_release(&put_batch); - rreq->iter.count += added; - rreq->submitted += added; - return 0; -} - /** * netfs_readahead - Helper to manage a read request * @ractl: The description of the readahead request @@ -419,7 +350,8 @@ void netfs_readahead(struct readahead_control *ractl) netfs_rreq_expand(rreq, ractl); rreq->ractl = ractl; - if (netfs_prime_buffer(rreq) < 0) + rreq->submitted = rreq->start; + if (rolling_buffer_init(&rreq->buffer, rreq->debug_id, ITER_DEST) < 0) goto cleanup_free; netfs_read_to_pagecache(rreq); @@ -435,22 +367,18 @@ EXPORT_SYMBOL(netfs_readahead); /* * Create a rolling buffer with a single occupying folio. */ -static int netfs_create_singular_buffer(struct netfs_io_request *rreq, struct folio *folio) +static int netfs_create_singular_buffer(struct netfs_io_request *rreq, struct folio *folio, + unsigned int rollbuf_flags) { - struct folio_queue *folioq; + ssize_t added; - folioq = netfs_folioq_alloc(rreq->debug_id, GFP_KERNEL, - netfs_trace_folioq_alloc_read_sing); - if (!folioq) + if (rolling_buffer_init(&rreq->buffer, rreq->debug_id, ITER_DEST) < 0) return -ENOMEM; - folioq_append(folioq, folio); - BUG_ON(folioq_folio(folioq, 0) != folio); - BUG_ON(folioq_folio_order(folioq, 0) != folio_order(folio)); - rreq->buffer = folioq; - rreq->buffer_tail = folioq; - rreq->submitted = rreq->start + rreq->len; - iov_iter_folio_queue(&rreq->iter, ITER_DEST, folioq, 0, 0, rreq->len); + added = rolling_buffer_append(&rreq->buffer, folio, rollbuf_flags); + if (added < 0) + return added; + rreq->submitted = rreq->start + added; rreq->ractl = (struct readahead_control *)1UL; return 0; } @@ -518,7 +446,7 @@ static int netfs_read_gaps(struct file *file, struct folio *folio) } if (to < flen) bvec_set_folio(&bvec[i++], folio, flen - to, to); - iov_iter_bvec(&rreq->iter, ITER_DEST, bvec, i, rreq->len); + iov_iter_bvec(&rreq->buffer.iter, ITER_DEST, bvec, i, rreq->len); rreq->submitted = rreq->start + flen; netfs_read_to_pagecache(rreq); @@ -586,7 +514,7 @@ int netfs_read_folio(struct file *file, struct folio *folio) trace_netfs_read(rreq, rreq->start, rreq->len, netfs_read_trace_readpage); /* Set up the output buffer */ - ret = netfs_create_singular_buffer(rreq, folio); + ret = netfs_create_singular_buffer(rreq, folio, 0); if (ret < 0) goto discard; @@ -743,7 +671,7 @@ retry: trace_netfs_read(rreq, pos, len, netfs_read_trace_write_begin); /* Set up the output buffer */ - ret = netfs_create_singular_buffer(rreq, folio); + ret = netfs_create_singular_buffer(rreq, folio, 0); if (ret < 0) goto error_put; @@ -808,11 +736,10 @@ int netfs_prefetch_for_write(struct file *file, struct folio *folio, trace_netfs_read(rreq, start, flen, netfs_read_trace_prefetch_for_write); /* Set up the output buffer */ - ret = netfs_create_singular_buffer(rreq, folio); + ret = netfs_create_singular_buffer(rreq, folio, NETFS_ROLLBUF_PAGECACHE_MARK); if (ret < 0) goto error_put; - folioq_mark2(rreq->buffer, 0); netfs_read_to_pagecache(rreq); ret = netfs_wait_for_read(rreq); netfs_put_request(rreq, false, netfs_rreq_trace_put_return); diff --git a/fs/netfs/direct_read.c b/fs/netfs/direct_read.c index b1a66a6e6bc2d..a3f23adbae0f6 100644 --- a/fs/netfs/direct_read.c +++ b/fs/netfs/direct_read.c @@ -25,7 +25,7 @@ static void netfs_prepare_dio_read_iterator(struct netfs_io_subrequest *subreq) subreq->len = rsize; if (unlikely(rreq->io_streams[0].sreq_max_segs)) { - size_t limit = netfs_limit_iter(&rreq->iter, 0, rsize, + size_t limit = netfs_limit_iter(&rreq->buffer.iter, 0, rsize, rreq->io_streams[0].sreq_max_segs); if (limit < rsize) { @@ -36,9 +36,9 @@ static void netfs_prepare_dio_read_iterator(struct netfs_io_subrequest *subreq) trace_netfs_sreq(subreq, netfs_sreq_trace_prepare); - subreq->io_iter = rreq->iter; + subreq->io_iter = rreq->buffer.iter; iov_iter_truncate(&subreq->io_iter, subreq->len); - iov_iter_advance(&rreq->iter, subreq->len); + iov_iter_advance(&rreq->buffer.iter, subreq->len); } /* @@ -199,15 +199,15 @@ ssize_t netfs_unbuffered_read_iter_locked(struct kiocb *iocb, struct iov_iter *i * the request. */ if (user_backed_iter(iter)) { - ret = netfs_extract_user_iter(iter, rreq->len, &rreq->iter, 0); + ret = netfs_extract_user_iter(iter, rreq->len, &rreq->buffer.iter, 0); if (ret < 0) goto out; - rreq->direct_bv = (struct bio_vec *)rreq->iter.bvec; + rreq->direct_bv = (struct bio_vec *)rreq->buffer.iter.bvec; rreq->direct_bv_count = ret; rreq->direct_bv_unpin = iov_iter_extract_will_pin(iter); - rreq->len = iov_iter_count(&rreq->iter); + rreq->len = iov_iter_count(&rreq->buffer.iter); } else { - rreq->iter = *iter; + rreq->buffer.iter = *iter; rreq->len = orig_count; rreq->direct_bv_unpin = false; iov_iter_advance(iter, orig_count); diff --git a/fs/netfs/direct_write.c b/fs/netfs/direct_write.c index 173e8b5e6a930..eded8afaa60bd 100644 --- a/fs/netfs/direct_write.c +++ b/fs/netfs/direct_write.c @@ -68,19 +68,17 @@ ssize_t netfs_unbuffered_write_iter_locked(struct kiocb *iocb, struct iov_iter * * request. */ if (async || user_backed_iter(iter)) { - n = netfs_extract_user_iter(iter, len, &wreq->iter, 0); + n = netfs_extract_user_iter(iter, len, &wreq->buffer.iter, 0); if (n < 0) { ret = n; goto out; } - wreq->direct_bv = (struct bio_vec *)wreq->iter.bvec; + wreq->direct_bv = (struct bio_vec *)wreq->buffer.iter.bvec; wreq->direct_bv_count = n; wreq->direct_bv_unpin = iov_iter_extract_will_pin(iter); } else { - wreq->iter = *iter; + wreq->buffer.iter = *iter; } - - wreq->io_iter = wreq->iter; } __set_bit(NETFS_RREQ_USE_IO_ITER, &wreq->flags); @@ -92,7 +90,7 @@ ssize_t netfs_unbuffered_write_iter_locked(struct kiocb *iocb, struct iov_iter * __set_bit(NETFS_RREQ_UPLOAD_TO_SERVER, &wreq->flags); if (async) wreq->iocb = iocb; - wreq->len = iov_iter_count(&wreq->io_iter); + wreq->len = iov_iter_count(&wreq->buffer.iter); wreq->cleanup = netfs_cleanup_dio_write; ret = netfs_unbuffered_write(wreq, is_sync_kiocb(iocb), wreq->len); if (ret < 0) { diff --git a/fs/netfs/internal.h b/fs/netfs/internal.h index 01b013f558f7f..ccd9058acb61f 100644 --- a/fs/netfs/internal.h +++ b/fs/netfs/internal.h @@ -60,10 +60,6 @@ static inline void netfs_proc_del_rreq(struct netfs_io_request *rreq) {} */ struct folio_queue *netfs_buffer_make_space(struct netfs_io_request *rreq, enum netfs_folioq_trace trace); -int netfs_buffer_append_folio(struct netfs_io_request *rreq, struct folio *folio, - bool needs_put); -struct folio_queue *netfs_delete_buffer_head(struct netfs_io_request *wreq); -void netfs_clear_buffer(struct netfs_io_request *rreq); void netfs_reset_iter(struct netfs_io_subrequest *subreq); /* diff --git a/fs/netfs/misc.c b/fs/netfs/misc.c index afe032551de50..4249715f41715 100644 --- a/fs/netfs/misc.c +++ b/fs/netfs/misc.c @@ -8,153 +8,6 @@ #include #include "internal.h" -/** - * netfs_folioq_alloc - Allocate a folio_queue struct - * @rreq_id: Associated debugging ID for tracing purposes - * @gfp: Allocation constraints - * @trace: Trace tag to indicate the purpose of the allocation - * - * Allocate, initialise and account the folio_queue struct and log a trace line - * to mark the allocation. - */ -struct folio_queue *netfs_folioq_alloc(unsigned int rreq_id, gfp_t gfp, - unsigned int /*enum netfs_folioq_trace*/ trace) -{ - static atomic_t debug_ids; - struct folio_queue *fq; - - fq = kmalloc(sizeof(*fq), gfp); - if (fq) { - netfs_stat(&netfs_n_folioq); - folioq_init(fq, rreq_id); - fq->debug_id = atomic_inc_return(&debug_ids); - trace_netfs_folioq(fq, trace); - } - return fq; -} -EXPORT_SYMBOL(netfs_folioq_alloc); - -/** - * netfs_folioq_free - Free a folio_queue struct - * @folioq: The object to free - * @trace: Trace tag to indicate which free - * - * Free and unaccount the folio_queue struct. - */ -void netfs_folioq_free(struct folio_queue *folioq, - unsigned int /*enum netfs_trace_folioq*/ trace) -{ - trace_netfs_folioq(folioq, trace); - netfs_stat_d(&netfs_n_folioq); - kfree(folioq); -} -EXPORT_SYMBOL(netfs_folioq_free); - -/* - * Make sure there's space in the rolling queue. - */ -struct folio_queue *netfs_buffer_make_space(struct netfs_io_request *rreq, - enum netfs_folioq_trace trace) -{ - struct folio_queue *tail = rreq->buffer_tail, *prev; - unsigned int prev_nr_slots = 0; - - if (WARN_ON_ONCE(!rreq->buffer && tail) || - WARN_ON_ONCE(rreq->buffer && !tail)) - return ERR_PTR(-EIO); - - prev = tail; - if (prev) { - if (!folioq_full(tail)) - return tail; - prev_nr_slots = folioq_nr_slots(tail); - } - - tail = netfs_folioq_alloc(rreq->debug_id, GFP_NOFS, trace); - if (!tail) - return ERR_PTR(-ENOMEM); - tail->prev = prev; - if (prev) - /* [!] NOTE: After we set prev->next, the consumer is entirely - * at liberty to delete prev. - */ - WRITE_ONCE(prev->next, tail); - - rreq->buffer_tail = tail; - if (!rreq->buffer) { - rreq->buffer = tail; - iov_iter_folio_queue(&rreq->io_iter, ITER_SOURCE, tail, 0, 0, 0); - } else { - /* Make sure we don't leave the master iterator pointing to a - * block that might get immediately consumed. - */ - if (rreq->io_iter.folioq == prev && - rreq->io_iter.folioq_slot == prev_nr_slots) { - rreq->io_iter.folioq = tail; - rreq->io_iter.folioq_slot = 0; - } - } - rreq->buffer_tail_slot = 0; - return tail; -} - -/* - * Append a folio to the rolling queue. - */ -int netfs_buffer_append_folio(struct netfs_io_request *rreq, struct folio *folio, - bool needs_put) -{ - struct folio_queue *tail; - unsigned int slot, order = folio_order(folio); - - tail = netfs_buffer_make_space(rreq, netfs_trace_folioq_alloc_append_folio); - if (IS_ERR(tail)) - return PTR_ERR(tail); - - rreq->io_iter.count += PAGE_SIZE << order; - - slot = folioq_append(tail, folio); - /* Store the counter after setting the slot. */ - smp_store_release(&rreq->buffer_tail_slot, slot); - return 0; -} - -/* - * Delete the head of a rolling queue. - */ -struct folio_queue *netfs_delete_buffer_head(struct netfs_io_request *wreq) -{ - struct folio_queue *head = wreq->buffer, *next = head->next; - - if (next) - next->prev = NULL; - netfs_folioq_free(head, netfs_trace_folioq_delete); - wreq->buffer = next; - return next; -} - -/* - * Clear out a rolling queue. - */ -void netfs_clear_buffer(struct netfs_io_request *rreq) -{ - struct folio_queue *p; - - while ((p = rreq->buffer)) { - rreq->buffer = p->next; - for (int slot = 0; slot < folioq_count(p); slot++) { - struct folio *folio = folioq_folio(p, slot); - if (!folio) - continue; - if (folioq_is_marked(p, slot)) { - trace_netfs_folio(folio, netfs_folio_trace_put); - folio_put(folio); - } - } - netfs_folioq_free(p, netfs_trace_folioq_clear); - } -} - /* * Reset the subrequest iterator to refer just to the region remaining to be * read. The iterator may or may not have been advanced by socket ops or diff --git a/fs/netfs/objects.c b/fs/netfs/objects.c index 31e388ec6e487..5cdddaf1f9788 100644 --- a/fs/netfs/objects.c +++ b/fs/netfs/objects.c @@ -143,7 +143,7 @@ static void netfs_free_request(struct work_struct *work) } kvfree(rreq->direct_bv); } - netfs_clear_buffer(rreq); + rolling_buffer_clear(&rreq->buffer); if (atomic_dec_and_test(&ictx->io_count)) wake_up_var(&ictx->io_count); diff --git a/fs/netfs/read_pgpriv2.c b/fs/netfs/read_pgpriv2.c index 54d5004fec182..9eee5af6b327b 100644 --- a/fs/netfs/read_pgpriv2.c +++ b/fs/netfs/read_pgpriv2.c @@ -34,8 +34,9 @@ void netfs_pgpriv2_mark_copy_to_cache(struct netfs_io_subrequest *subreq, * [DEPRECATED] Cancel PG_private_2 on all marked folios in the event of an * unrecoverable error. */ -static void netfs_pgpriv2_cancel(struct folio_queue *folioq) +static void netfs_pgpriv2_cancel(struct rolling_buffer *buffer) { + struct folio_queue *folioq = buffer->tail; struct folio *folio; int slot; @@ -94,7 +95,7 @@ static int netfs_pgpriv2_copy_folio(struct netfs_io_request *wreq, struct folio trace_netfs_folio(folio, netfs_folio_trace_store_copy); /* Attach the folio to the rolling buffer. */ - if (netfs_buffer_append_folio(wreq, folio, false) < 0) + if (rolling_buffer_append(&wreq->buffer, folio, 0) < 0) return -ENOMEM; cache->submit_extendable_to = fsize; @@ -109,7 +110,7 @@ static int netfs_pgpriv2_copy_folio(struct netfs_io_request *wreq, struct folio do { ssize_t part; - wreq->io_iter.iov_offset = cache->submit_off; + wreq->buffer.iter.iov_offset = cache->submit_off; atomic64_set(&wreq->issued_to, fpos + cache->submit_off); cache->submit_extendable_to = fsize - cache->submit_off; @@ -122,8 +123,8 @@ static int netfs_pgpriv2_copy_folio(struct netfs_io_request *wreq, struct folio cache->submit_len -= part; } while (cache->submit_len > 0); - wreq->io_iter.iov_offset = 0; - iov_iter_advance(&wreq->io_iter, fsize); + wreq->buffer.iter.iov_offset = 0; + rolling_buffer_advance(&wreq->buffer, fsize); atomic64_set(&wreq->issued_to, fpos + fsize); if (flen < fsize) @@ -151,7 +152,7 @@ void netfs_pgpriv2_write_to_the_cache(struct netfs_io_request *rreq) goto couldnt_start; /* Need the first folio to be able to set up the op. */ - for (folioq = rreq->buffer; folioq; folioq = folioq->next) { + for (folioq = rreq->buffer.tail; folioq; folioq = folioq->next) { if (folioq->marks3) { slot = __ffs(folioq->marks3); break; @@ -198,7 +199,7 @@ void netfs_pgpriv2_write_to_the_cache(struct netfs_io_request *rreq) netfs_put_request(wreq, false, netfs_rreq_trace_put_return); _leave(" = %d", error); couldnt_start: - netfs_pgpriv2_cancel(rreq->buffer); + netfs_pgpriv2_cancel(&rreq->buffer); } /* @@ -207,13 +208,13 @@ couldnt_start: */ bool netfs_pgpriv2_unlock_copied_folios(struct netfs_io_request *wreq) { - struct folio_queue *folioq = wreq->buffer; + struct folio_queue *folioq = wreq->buffer.tail; unsigned long long collected_to = wreq->collected_to; - unsigned int slot = wreq->buffer_head_slot; + unsigned int slot = wreq->buffer.first_tail_slot; bool made_progress = false; if (slot >= folioq_nr_slots(folioq)) { - folioq = netfs_delete_buffer_head(wreq); + folioq = rolling_buffer_delete_spent(&wreq->buffer); slot = 0; } @@ -252,9 +253,9 @@ bool netfs_pgpriv2_unlock_copied_folios(struct netfs_io_request *wreq) folioq_clear(folioq, slot); slot++; if (slot >= folioq_nr_slots(folioq)) { - if (READ_ONCE(wreq->buffer_tail) == folioq) - break; - folioq = netfs_delete_buffer_head(wreq); + folioq = rolling_buffer_delete_spent(&wreq->buffer); + if (!folioq) + goto done; slot = 0; } @@ -262,7 +263,8 @@ bool netfs_pgpriv2_unlock_copied_folios(struct netfs_io_request *wreq) break; } - wreq->buffer = folioq; - wreq->buffer_head_slot = slot; + wreq->buffer.tail = folioq; +done: + wreq->buffer.first_tail_slot = slot; return made_progress; } diff --git a/fs/netfs/read_retry.c b/fs/netfs/read_retry.c index 21b4a54e545e1..0983234c2183f 100644 --- a/fs/netfs/read_retry.c +++ b/fs/netfs/read_retry.c @@ -245,7 +245,7 @@ void netfs_unlock_abandoned_read_pages(struct netfs_io_request *rreq) { struct folio_queue *p; - for (p = rreq->buffer; p; p = p->next) { + for (p = rreq->buffer.tail; p; p = p->next) { for (int slot = 0; slot < folioq_count(p); slot++) { struct folio *folio = folioq_folio(p, slot); diff --git a/fs/netfs/rolling_buffer.c b/fs/netfs/rolling_buffer.c new file mode 100644 index 0000000000000..75d97af14b4ab --- /dev/null +++ b/fs/netfs/rolling_buffer.c @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Rolling buffer helpers + * + * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include +#include "internal.h" + +static atomic_t debug_ids; + +/** + * netfs_folioq_alloc - Allocate a folio_queue struct + * @rreq_id: Associated debugging ID for tracing purposes + * @gfp: Allocation constraints + * @trace: Trace tag to indicate the purpose of the allocation + * + * Allocate, initialise and account the folio_queue struct and log a trace line + * to mark the allocation. + */ +struct folio_queue *netfs_folioq_alloc(unsigned int rreq_id, gfp_t gfp, + unsigned int /*enum netfs_folioq_trace*/ trace) +{ + struct folio_queue *fq; + + fq = kmalloc(sizeof(*fq), gfp); + if (fq) { + netfs_stat(&netfs_n_folioq); + folioq_init(fq, rreq_id); + fq->debug_id = atomic_inc_return(&debug_ids); + trace_netfs_folioq(fq, trace); + } + return fq; +} +EXPORT_SYMBOL(netfs_folioq_alloc); + +/** + * netfs_folioq_free - Free a folio_queue struct + * @folioq: The object to free + * @trace: Trace tag to indicate which free + * + * Free and unaccount the folio_queue struct. + */ +void netfs_folioq_free(struct folio_queue *folioq, + unsigned int /*enum netfs_trace_folioq*/ trace) +{ + trace_netfs_folioq(folioq, trace); + netfs_stat_d(&netfs_n_folioq); + kfree(folioq); +} +EXPORT_SYMBOL(netfs_folioq_free); + +/* + * Initialise a rolling buffer. We allocate an empty folio queue struct to so + * that the pointers can be independently driven by the producer and the + * consumer. + */ +int rolling_buffer_init(struct rolling_buffer *roll, unsigned int rreq_id, + unsigned int direction) +{ + struct folio_queue *fq; + + fq = netfs_folioq_alloc(rreq_id, GFP_NOFS, netfs_trace_folioq_rollbuf_init); + if (!fq) + return -ENOMEM; + + roll->head = fq; + roll->tail = fq; + iov_iter_folio_queue(&roll->iter, direction, fq, 0, 0, 0); + return 0; +} + +/* + * Add another folio_queue to a rolling buffer if there's no space left. + */ +int rolling_buffer_make_space(struct rolling_buffer *roll) +{ + struct folio_queue *fq, *head = roll->head; + + if (!folioq_full(head)) + return 0; + + fq = netfs_folioq_alloc(head->rreq_id, GFP_NOFS, netfs_trace_folioq_make_space); + if (!fq) + return -ENOMEM; + fq->prev = head; + + roll->head = fq; + if (folioq_full(head)) { + /* Make sure we don't leave the master iterator pointing to a + * block that might get immediately consumed. + */ + if (roll->iter.folioq == head && + roll->iter.folioq_slot == folioq_nr_slots(head)) { + roll->iter.folioq = fq; + roll->iter.folioq_slot = 0; + } + } + + /* Make sure the initialisation is stored before the next pointer. + * + * [!] NOTE: After we set head->next, the consumer is at liberty to + * immediately delete the old head. + */ + smp_store_release(&head->next, fq); + return 0; +} + +/* + * Decant the list of folios to read into a rolling buffer. + */ +ssize_t rolling_buffer_load_from_ra(struct rolling_buffer *roll, + struct readahead_control *ractl, + struct folio_batch *put_batch) +{ + struct folio_queue *fq; + struct page **vec; + int nr, ix, to; + ssize_t size = 0; + + if (rolling_buffer_make_space(roll) < 0) + return -ENOMEM; + + fq = roll->head; + vec = (struct page **)fq->vec.folios; + nr = __readahead_batch(ractl, vec + folio_batch_count(&fq->vec), + folio_batch_space(&fq->vec)); + ix = fq->vec.nr; + to = ix + nr; + fq->vec.nr = to; + for (; ix < to; ix++) { + struct folio *folio = folioq_folio(fq, ix); + unsigned int order = folio_order(folio); + + fq->orders[ix] = order; + size += PAGE_SIZE << order; + trace_netfs_folio(folio, netfs_folio_trace_read); + if (!folio_batch_add(put_batch, folio)) + folio_batch_release(put_batch); + } + WRITE_ONCE(roll->iter.count, roll->iter.count + size); + + /* Store the counter after setting the slot. */ + smp_store_release(&roll->next_head_slot, to); + + for (; ix < folioq_nr_slots(fq); ix++) + folioq_clear(fq, ix); + + return size; +} + +/* + * Append a folio to the rolling buffer. + */ +ssize_t rolling_buffer_append(struct rolling_buffer *roll, struct folio *folio, + unsigned int flags) +{ + ssize_t size = folio_size(folio); + int slot; + + if (rolling_buffer_make_space(roll) < 0) + return -ENOMEM; + + slot = folioq_append(roll->head, folio); + if (flags & ROLLBUF_MARK_1) + folioq_mark(roll->head, slot); + if (flags & ROLLBUF_MARK_2) + folioq_mark2(roll->head, slot); + + WRITE_ONCE(roll->iter.count, roll->iter.count + size); + + /* Store the counter after setting the slot. */ + smp_store_release(&roll->next_head_slot, slot); + return size; +} + +/* + * Delete a spent buffer from a rolling queue and return the next in line. We + * don't return the last buffer to keep the pointers independent, but return + * NULL instead. + */ +struct folio_queue *rolling_buffer_delete_spent(struct rolling_buffer *roll) +{ + struct folio_queue *spent = roll->tail, *next = READ_ONCE(spent->next); + + if (!next) + return NULL; + next->prev = NULL; + netfs_folioq_free(spent, netfs_trace_folioq_delete); + roll->tail = next; + return next; +} + +/* + * Clear out a rolling queue. Folios that have mark 1 set are put. + */ +void rolling_buffer_clear(struct rolling_buffer *roll) +{ + struct folio_batch fbatch; + struct folio_queue *p; + + folio_batch_init(&fbatch); + + while ((p = roll->tail)) { + roll->tail = p->next; + for (int slot = 0; slot < folioq_count(p); slot++) { + struct folio *folio = folioq_folio(p, slot); + + if (!folio) + continue; + if (folioq_is_marked(p, slot)) { + trace_netfs_folio(folio, netfs_folio_trace_put); + if (!folio_batch_add(&fbatch, folio)) + folio_batch_release(&fbatch); + } + } + + netfs_folioq_free(p, netfs_trace_folioq_clear); + } + + folio_batch_release(&fbatch); +} diff --git a/fs/netfs/write_collect.c b/fs/netfs/write_collect.c index ca3a11ed9b541..364c1f9d58152 100644 --- a/fs/netfs/write_collect.c +++ b/fs/netfs/write_collect.c @@ -83,9 +83,9 @@ end_wb: static void netfs_writeback_unlock_folios(struct netfs_io_request *wreq, unsigned int *notes) { - struct folio_queue *folioq = wreq->buffer; + struct folio_queue *folioq = wreq->buffer.tail; unsigned long long collected_to = wreq->collected_to; - unsigned int slot = wreq->buffer_head_slot; + unsigned int slot = wreq->buffer.first_tail_slot; if (wreq->origin == NETFS_PGPRIV2_COPY_TO_CACHE) { if (netfs_pgpriv2_unlock_copied_folios(wreq)) @@ -94,7 +94,9 @@ static void netfs_writeback_unlock_folios(struct netfs_io_request *wreq, } if (slot >= folioq_nr_slots(folioq)) { - folioq = netfs_delete_buffer_head(wreq); + folioq = rolling_buffer_delete_spent(&wreq->buffer); + if (!folioq) + return; slot = 0; } @@ -134,9 +136,9 @@ static void netfs_writeback_unlock_folios(struct netfs_io_request *wreq, folioq_clear(folioq, slot); slot++; if (slot >= folioq_nr_slots(folioq)) { - if (READ_ONCE(wreq->buffer_tail) == folioq) - break; - folioq = netfs_delete_buffer_head(wreq); + folioq = rolling_buffer_delete_spent(&wreq->buffer); + if (!folioq) + goto done; slot = 0; } @@ -144,8 +146,9 @@ static void netfs_writeback_unlock_folios(struct netfs_io_request *wreq, break; } - wreq->buffer = folioq; - wreq->buffer_head_slot = slot; + wreq->buffer.tail = folioq; +done: + wreq->buffer.first_tail_slot = slot; } /* diff --git a/fs/netfs/write_issue.c b/fs/netfs/write_issue.c index 87e5cf4a09576..88ceba49ff693 100644 --- a/fs/netfs/write_issue.c +++ b/fs/netfs/write_issue.c @@ -107,6 +107,8 @@ struct netfs_io_request *netfs_create_write_req(struct address_space *mapping, ictx = netfs_inode(wreq->inode); if (is_buffered && netfs_is_cache_enabled(ictx)) fscache_begin_write_operation(&wreq->cache_resources, netfs_i_cookie(ictx)); + if (rolling_buffer_init(&wreq->buffer, wreq->debug_id, ITER_SOURCE) < 0) + goto nomem; wreq->cleaned_to = wreq->start; @@ -129,6 +131,10 @@ struct netfs_io_request *netfs_create_write_req(struct address_space *mapping, } return wreq; +nomem: + wreq->error = -ENOMEM; + netfs_put_request(wreq, false, netfs_rreq_trace_put_failed); + return ERR_PTR(-ENOMEM); } /** @@ -153,16 +159,15 @@ static void netfs_prepare_write(struct netfs_io_request *wreq, loff_t start) { struct netfs_io_subrequest *subreq; - struct iov_iter *wreq_iter = &wreq->io_iter; + struct iov_iter *wreq_iter = &wreq->buffer.iter; /* Make sure we don't point the iterator at a used-up folio_queue * struct being used as a placeholder to prevent the queue from * collapsing. In such a case, extend the queue. */ if (iov_iter_is_folioq(wreq_iter) && - wreq_iter->folioq_slot >= folioq_nr_slots(wreq_iter->folioq)) { - netfs_buffer_make_space(wreq, netfs_trace_folioq_prep_write); - } + wreq_iter->folioq_slot >= folioq_nr_slots(wreq_iter->folioq)) + rolling_buffer_make_space(&wreq->buffer); subreq = netfs_alloc_subrequest(wreq); subreq->source = stream->source; @@ -327,6 +332,9 @@ static int netfs_write_folio(struct netfs_io_request *wreq, _enter(""); + if (rolling_buffer_make_space(&wreq->buffer) < 0) + return -ENOMEM; + /* netfs_perform_write() may shift i_size around the page or from out * of the page to beyond it, but cannot move i_size into or through the * page since we have it locked. @@ -431,7 +439,7 @@ static int netfs_write_folio(struct netfs_io_request *wreq, } /* Attach the folio to the rolling buffer. */ - netfs_buffer_append_folio(wreq, folio, false); + rolling_buffer_append(&wreq->buffer, folio, 0); /* Move the submission point forward to allow for write-streaming data * not starting at the front of the page. We don't do write-streaming @@ -478,7 +486,7 @@ static int netfs_write_folio(struct netfs_io_request *wreq, /* Advance the iterator(s). */ if (stream->submit_off > iter_off) { - iov_iter_advance(&wreq->io_iter, stream->submit_off - iter_off); + rolling_buffer_advance(&wreq->buffer, stream->submit_off - iter_off); iter_off = stream->submit_off; } @@ -496,7 +504,7 @@ static int netfs_write_folio(struct netfs_io_request *wreq, } if (fsize > iter_off) - iov_iter_advance(&wreq->io_iter, fsize - iter_off); + rolling_buffer_advance(&wreq->buffer, fsize - iter_off); atomic64_set(&wreq->issued_to, fpos + fsize); if (!debug) @@ -635,7 +643,7 @@ int netfs_advance_writethrough(struct netfs_io_request *wreq, struct writeback_c struct folio **writethrough_cache) { _enter("R=%x ic=%zu ws=%u cp=%zu tp=%u", - wreq->debug_id, wreq->iter.count, wreq->wsize, copied, to_page_end); + wreq->debug_id, wreq->buffer.iter.count, wreq->wsize, copied, to_page_end); if (!*writethrough_cache) { if (folio_test_dirty(folio)) @@ -710,7 +718,7 @@ int netfs_unbuffered_write(struct netfs_io_request *wreq, bool may_wait, size_t part = netfs_advance_write(wreq, upload, start, len, false); start += part; len -= part; - iov_iter_advance(&wreq->io_iter, part); + rolling_buffer_advance(&wreq->buffer, part); if (test_bit(NETFS_RREQ_PAUSE, &wreq->flags)) { trace_netfs_rreq(wreq, netfs_rreq_trace_wait_pause); wait_on_bit(&wreq->flags, NETFS_RREQ_PAUSE, TASK_UNINTERRUPTIBLE); diff --git a/include/linux/netfs.h b/include/linux/netfs.h index 5b2f427f8e3eb..bd922f0936e34 100644 --- a/include/linux/netfs.h +++ b/include/linux/netfs.h @@ -18,6 +18,7 @@ #include #include #include +#include enum netfs_sreq_ref_trace; typedef struct mempool_s mempool_t; @@ -238,10 +239,9 @@ struct netfs_io_request { struct netfs_io_stream io_streams[2]; /* Streams of parallel I/O operations */ #define NR_IO_STREAMS 2 //wreq->nr_io_streams struct netfs_group *group; /* Writeback group being written back */ - struct folio_queue *buffer; /* Head of I/O buffer */ - struct folio_queue *buffer_tail; /* Tail of I/O buffer */ - struct iov_iter iter; /* Unencrypted-side iterator */ - struct iov_iter io_iter; /* I/O (Encrypted-side) iterator */ + struct rolling_buffer buffer; /* Unencrypted buffer */ +#define NETFS_ROLLBUF_PUT_MARK ROLLBUF_MARK_1 +#define NETFS_ROLLBUF_PAGECACHE_MARK ROLLBUF_MARK_2 void *netfs_priv; /* Private data for the netfs */ void *netfs_priv2; /* Private data for the netfs */ struct bio_vec *direct_bv; /* DIO buffer list (when handling iovec-iter) */ @@ -259,8 +259,6 @@ struct netfs_io_request { long error; /* 0 or error that occurred */ enum netfs_io_origin origin; /* Origin of the request */ bool direct_bv_unpin; /* T if direct_bv[] must be unpinned */ - u8 buffer_head_slot; /* First slot in ->buffer */ - u8 buffer_tail_slot; /* Next slot in ->buffer_tail */ unsigned long long i_size; /* Size of the file */ unsigned long long start; /* Start position */ atomic64_t issued_to; /* Write issuer folio cursor */ diff --git a/include/linux/rolling_buffer.h b/include/linux/rolling_buffer.h new file mode 100644 index 0000000000000..ac15b1ffdd831 --- /dev/null +++ b/include/linux/rolling_buffer.h @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Rolling buffer of folios + * + * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#ifndef _ROLLING_BUFFER_H +#define _ROLLING_BUFFER_H + +#include +#include + +/* + * Rolling buffer. Whilst the buffer is live and in use, folios and folio + * queue segments can be added to one end by one thread and removed from the + * other end by another thread. The buffer isn't allowed to be empty; it must + * always have at least one folio_queue in it so that neither side has to + * modify both queue pointers. + * + * The iterator in the buffer is extended as buffers are inserted. It can be + * snapshotted to use a segment of the buffer. + */ +struct rolling_buffer { + struct folio_queue *head; /* Producer's insertion point */ + struct folio_queue *tail; /* Consumer's removal point */ + struct iov_iter iter; /* Iterator tracking what's left in the buffer */ + u8 next_head_slot; /* Next slot in ->head */ + u8 first_tail_slot; /* First slot in ->tail */ +}; + +/* + * Snapshot of a rolling buffer. + */ +struct rolling_buffer_snapshot { + struct folio_queue *curr_folioq; /* Queue segment in which current folio resides */ + unsigned char curr_slot; /* Folio currently being read */ + unsigned char curr_order; /* Order of folio */ +}; + +/* Marks to store per-folio in the internal folio_queue structs. */ +#define ROLLBUF_MARK_1 BIT(0) +#define ROLLBUF_MARK_2 BIT(1) + +int rolling_buffer_init(struct rolling_buffer *roll, unsigned int rreq_id, + unsigned int direction); +int rolling_buffer_make_space(struct rolling_buffer *roll); +ssize_t rolling_buffer_load_from_ra(struct rolling_buffer *roll, + struct readahead_control *ractl, + struct folio_batch *put_batch); +ssize_t rolling_buffer_append(struct rolling_buffer *roll, struct folio *folio, + unsigned int flags); +struct folio_queue *rolling_buffer_delete_spent(struct rolling_buffer *roll); +void rolling_buffer_clear(struct rolling_buffer *roll); + +static inline void rolling_buffer_advance(struct rolling_buffer *roll, size_t amount) +{ + iov_iter_advance(&roll->iter, amount); +} + +#endif /* _ROLLING_BUFFER_H */ diff --git a/include/trace/events/netfs.h b/include/trace/events/netfs.h index 50aa6745df95f..2dfc9f716e3b0 100644 --- a/include/trace/events/netfs.h +++ b/include/trace/events/netfs.h @@ -198,7 +198,9 @@ EM(netfs_trace_folioq_alloc_read_sing, "alloc-r-sing") \ EM(netfs_trace_folioq_clear, "clear") \ EM(netfs_trace_folioq_delete, "delete") \ + EM(netfs_trace_folioq_make_space, "make-space") \ EM(netfs_trace_folioq_prep_write, "prep-wr") \ + EM(netfs_trace_folioq_rollbuf_init, "roll-init") \ E_(netfs_trace_folioq_read_progress, "r-progress") #ifndef __NETFS_DECLARE_TRACE_ENUMS_ONCE_ONLY -- cgit v1.2.3 From e61bfaad8fd86ac84eac633e0bbaac47a5dfd358 Mon Sep 17 00:00:00 2001 From: David Howells Date: Mon, 16 Dec 2024 20:41:08 +0000 Subject: netfs: Add functions to build/clean a buffer in a folio_queue Add two netfslib functions to build up or clean up a buffer in a folio_queue. The first, netfs_alloc_folioq_buffer() will add folios to a buffer, extending up at least to the given size. If it can, it will add multipage folios. The folios are optionally have the mapping set and will have the index set according to the distance from the front of the folio queue. The second function will free up a folio queue and put any folios in the queue that have the first mark set. The netfs_folio tracepoint is also altered to cope with folios that have a NULL mapping, and the folios being added/put will have trace lines emitted and will be accounted in the stats. Signed-off-by: David Howells Link: https://lore.kernel.org/r/20241216204124.3752367-19-dhowells@redhat.com cc: Jeff Layton cc: Marc Dionne cc: netfs@lists.linux.dev cc: linux-afs@lists.infradead.org cc: linux-fsdevel@vger.kernel.org Signed-off-by: Christian Brauner --- fs/netfs/misc.c | 96 ++++++++++++++++++++++++++++++++++++++++++++ include/linux/netfs.h | 6 +++ include/trace/events/netfs.h | 6 +-- 3 files changed, 104 insertions(+), 4 deletions(-) (limited to 'fs/netfs/misc.c') diff --git a/fs/netfs/misc.c b/fs/netfs/misc.c index 4249715f41715..7099aa07737ac 100644 --- a/fs/netfs/misc.c +++ b/fs/netfs/misc.c @@ -8,6 +8,102 @@ #include #include "internal.h" +/** + * netfs_alloc_folioq_buffer - Allocate buffer space into a folio queue + * @mapping: Address space to set on the folio (or NULL). + * @_buffer: Pointer to the folio queue to add to (may point to a NULL; updated). + * @_cur_size: Current size of the buffer (updated). + * @size: Target size of the buffer. + * @gfp: The allocation constraints. + */ +int netfs_alloc_folioq_buffer(struct address_space *mapping, + struct folio_queue **_buffer, + size_t *_cur_size, ssize_t size, gfp_t gfp) +{ + struct folio_queue *tail = *_buffer, *p; + + size = round_up(size, PAGE_SIZE); + if (*_cur_size >= size) + return 0; + + if (tail) + while (tail->next) + tail = tail->next; + + do { + struct folio *folio; + int order = 0, slot; + + if (!tail || folioq_full(tail)) { + p = netfs_folioq_alloc(0, GFP_NOFS, netfs_trace_folioq_alloc_buffer); + if (!p) + return -ENOMEM; + if (tail) { + tail->next = p; + p->prev = tail; + } else { + *_buffer = p; + } + tail = p; + } + + if (size - *_cur_size > PAGE_SIZE) + order = umin(ilog2(size - *_cur_size) - PAGE_SHIFT, + MAX_PAGECACHE_ORDER); + + folio = folio_alloc(gfp, order); + if (!folio && order > 0) + folio = folio_alloc(gfp, 0); + if (!folio) + return -ENOMEM; + + folio->mapping = mapping; + folio->index = *_cur_size / PAGE_SIZE; + trace_netfs_folio(folio, netfs_folio_trace_alloc_buffer); + slot = folioq_append_mark(tail, folio); + *_cur_size += folioq_folio_size(tail, slot); + } while (*_cur_size < size); + + return 0; +} +EXPORT_SYMBOL(netfs_alloc_folioq_buffer); + +/** + * netfs_free_folioq_buffer - Free a folio queue. + * @fq: The start of the folio queue to free + * + * Free up a chain of folio_queues and, if marked, the marked folios they point + * to. + */ +void netfs_free_folioq_buffer(struct folio_queue *fq) +{ + struct folio_queue *next; + struct folio_batch fbatch; + + folio_batch_init(&fbatch); + + for (; fq; fq = next) { + for (int slot = 0; slot < folioq_count(fq); slot++) { + struct folio *folio = folioq_folio(fq, slot); + + if (!folio || + !folioq_is_marked(fq, slot)) + continue; + + trace_netfs_folio(folio, netfs_folio_trace_put); + if (folio_batch_add(&fbatch, folio)) + folio_batch_release(&fbatch); + } + + netfs_stat_d(&netfs_n_folioq); + next = fq->next; + kfree(fq); + } + + folio_batch_release(&fbatch); +} +EXPORT_SYMBOL(netfs_free_folioq_buffer); + /* * Reset the subrequest iterator to refer just to the region remaining to be * read. The iterator may or may not have been advanced by socket ops or diff --git a/include/linux/netfs.h b/include/linux/netfs.h index 374e54beacbef..dd737344cff3a 100644 --- a/include/linux/netfs.h +++ b/include/linux/netfs.h @@ -457,6 +457,12 @@ struct folio_queue *netfs_folioq_alloc(unsigned int rreq_id, gfp_t gfp, void netfs_folioq_free(struct folio_queue *folioq, unsigned int trace /*enum netfs_trace_folioq*/); +/* Buffer wrangling helpers API. */ +int netfs_alloc_folioq_buffer(struct address_space *mapping, + struct folio_queue **_buffer, + size_t *_cur_size, ssize_t size, gfp_t gfp); +void netfs_free_folioq_buffer(struct folio_queue *fq); + /** * netfs_inode - Get the netfs inode context from the inode * @inode: The inode to query diff --git a/include/trace/events/netfs.h b/include/trace/events/netfs.h index 02f6e179b7bc3..fc237ff23a33e 100644 --- a/include/trace/events/netfs.h +++ b/include/trace/events/netfs.h @@ -155,6 +155,7 @@ EM(netfs_streaming_filled_page, "mod-streamw-f") \ EM(netfs_streaming_cont_filled_page, "mod-streamw-f+") \ EM(netfs_folio_trace_abandon, "abandon") \ + EM(netfs_folio_trace_alloc_buffer, "alloc-buf") \ EM(netfs_folio_trace_cancel_copy, "cancel-copy") \ EM(netfs_folio_trace_cancel_store, "cancel-store") \ EM(netfs_folio_trace_clear, "clear") \ @@ -195,10 +196,7 @@ E_(netfs_trace_donate_to_deferred_next, "defer-next") #define netfs_folioq_traces \ - EM(netfs_trace_folioq_alloc_append_folio, "alloc-apf") \ - EM(netfs_trace_folioq_alloc_read_prep, "alloc-r-prep") \ - EM(netfs_trace_folioq_alloc_read_prime, "alloc-r-prime") \ - EM(netfs_trace_folioq_alloc_read_sing, "alloc-r-sing") \ + EM(netfs_trace_folioq_alloc_buffer, "alloc-buf") \ EM(netfs_trace_folioq_clear, "clear") \ EM(netfs_trace_folioq_delete, "delete") \ EM(netfs_trace_folioq_make_space, "make-space") \ -- cgit v1.2.3