diff options
author | Jakub Kicinski <kuba@kernel.org> | 2025-07-11 16:01:23 -0700 |
---|---|---|
committer | Jakub Kicinski <kuba@kernel.org> | 2025-07-11 16:01:24 -0700 |
commit | f45b45cbfae394583394051444f766fff3c151f5 (patch) | |
tree | 1677bf66c672baff68125bdada25b6c3965b9334 | |
parent | 0346000aaab8cf8baf4ae40c48a5779a03971e80 (diff) | |
parent | 1f376373bd225c90381b745e38fa65a9386f7f8e (diff) |
Merge branch 'net_sched-act-extend-rcu-use-in-dump-methods'
Eric Dumazet says:
====================
net_sched: act: extend RCU use in dump() methods
We are trying to get away from central RTNL in favor of fine-grained
mutexes. While looking at net/sched, I found that act already uses
RCU in the fast path for the most cases, and could also be used
in dump() methods.
This series is not complete and will be followed by a second one.
v1: https://lore.kernel.org/20250707130110.619822-1-edumazet@google.com
====================
Link: https://patch.msgid.link/20250709090204.797558-1-edumazet@google.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
-rw-r--r-- | include/net/act_api.h | 23 | ||||
-rw-r--r-- | include/net/tc_act/tc_connmark.h | 1 | ||||
-rw-r--r-- | include/net/tc_act/tc_csum.h | 1 | ||||
-rw-r--r-- | include/net/tc_act/tc_ct.h | 2 | ||||
-rw-r--r-- | include/net/tc_act/tc_ctinfo.h | 7 | ||||
-rw-r--r-- | include/net/tc_act/tc_mpls.h | 1 | ||||
-rw-r--r-- | include/net/tc_act/tc_nat.h | 1 | ||||
-rw-r--r-- | include/net/tc_act/tc_pedit.h | 1 | ||||
-rw-r--r-- | include/net/tc_act/tc_police.h | 3 | ||||
-rw-r--r-- | include/net/tc_act/tc_skbedit.h | 1 | ||||
-rw-r--r-- | net/sched/act_connmark.c | 18 | ||||
-rw-r--r-- | net/sched/act_csum.c | 18 | ||||
-rw-r--r-- | net/sched/act_ct.c | 30 | ||||
-rw-r--r-- | net/sched/act_ctinfo.c | 42 | ||||
-rw-r--r-- | net/sched/act_mpls.c | 21 | ||||
-rw-r--r-- | net/sched/act_nat.c | 25 | ||||
-rw-r--r-- | net/sched/act_pedit.c | 20 | ||||
-rw-r--r-- | net/sched/act_police.c | 18 | ||||
-rw-r--r-- | net/sched/act_skbedit.c | 20 |
19 files changed, 133 insertions, 120 deletions
diff --git a/include/net/act_api.h b/include/net/act_api.h index 04781c92b43d..2894cfff2da3 100644 --- a/include/net/act_api.h +++ b/include/net/act_api.h @@ -76,19 +76,24 @@ static inline void tcf_lastuse_update(struct tcf_t *tm) { unsigned long now = jiffies; - if (tm->lastuse != now) - tm->lastuse = now; - if (unlikely(!tm->firstuse)) - tm->firstuse = now; + if (READ_ONCE(tm->lastuse) != now) + WRITE_ONCE(tm->lastuse, now); + if (unlikely(!READ_ONCE(tm->firstuse))) + WRITE_ONCE(tm->firstuse, now); } static inline void tcf_tm_dump(struct tcf_t *dtm, const struct tcf_t *stm) { - dtm->install = jiffies_to_clock_t(jiffies - stm->install); - dtm->lastuse = jiffies_to_clock_t(jiffies - stm->lastuse); - dtm->firstuse = stm->firstuse ? - jiffies_to_clock_t(jiffies - stm->firstuse) : 0; - dtm->expires = jiffies_to_clock_t(stm->expires); + unsigned long firstuse, now = jiffies; + + dtm->install = jiffies_to_clock_t(now - READ_ONCE(stm->install)); + dtm->lastuse = jiffies_to_clock_t(now - READ_ONCE(stm->lastuse)); + + firstuse = READ_ONCE(stm->firstuse); + dtm->firstuse = firstuse ? + jiffies_to_clock_t(now - firstuse) : 0; + + dtm->expires = jiffies_to_clock_t(READ_ONCE(stm->expires)); } static inline enum flow_action_hw_stats tc_act_hw_stats(u8 hw_stats) diff --git a/include/net/tc_act/tc_connmark.h b/include/net/tc_act/tc_connmark.h index e8dd77a96748..a5ce83f3eea4 100644 --- a/include/net/tc_act/tc_connmark.h +++ b/include/net/tc_act/tc_connmark.h @@ -7,6 +7,7 @@ struct tcf_connmark_parms { struct net *net; u16 zone; + int action; struct rcu_head rcu; }; diff --git a/include/net/tc_act/tc_csum.h b/include/net/tc_act/tc_csum.h index 2515da0142a6..8d0c7a9f9345 100644 --- a/include/net/tc_act/tc_csum.h +++ b/include/net/tc_act/tc_csum.h @@ -8,6 +8,7 @@ struct tcf_csum_params { u32 update_flags; + int action; struct rcu_head rcu; }; diff --git a/include/net/tc_act/tc_ct.h b/include/net/tc_act/tc_ct.h index e6b45cb27ebf..8b90c86c0b0d 100644 --- a/include/net/tc_act/tc_ct.h +++ b/include/net/tc_act/tc_ct.h @@ -13,7 +13,7 @@ struct tcf_ct_params { struct nf_conntrack_helper *helper; struct nf_conn *tmpl; u16 zone; - + int action; u32 mark; u32 mark_mask; diff --git a/include/net/tc_act/tc_ctinfo.h b/include/net/tc_act/tc_ctinfo.h index f071c1d70a25..7fe01ab236da 100644 --- a/include/net/tc_act/tc_ctinfo.h +++ b/include/net/tc_act/tc_ctinfo.h @@ -7,6 +7,7 @@ struct tcf_ctinfo_params { struct rcu_head rcu; struct net *net; + int action; u32 dscpmask; u32 dscpstatemask; u32 cpmarkmask; @@ -18,9 +19,9 @@ struct tcf_ctinfo_params { struct tcf_ctinfo { struct tc_action common; struct tcf_ctinfo_params __rcu *params; - u64 stats_dscp_set; - u64 stats_dscp_error; - u64 stats_cpmark_set; + atomic64_t stats_dscp_set; + atomic64_t stats_dscp_error; + atomic64_t stats_cpmark_set; }; enum { diff --git a/include/net/tc_act/tc_mpls.h b/include/net/tc_act/tc_mpls.h index d452e5e94fd0..dd067bd4018d 100644 --- a/include/net/tc_act/tc_mpls.h +++ b/include/net/tc_act/tc_mpls.h @@ -10,6 +10,7 @@ struct tcf_mpls_params { int tcfm_action; u32 tcfm_label; + int action; /* tcf_action */ u8 tcfm_tc; u8 tcfm_ttl; u8 tcfm_bos; diff --git a/include/net/tc_act/tc_nat.h b/include/net/tc_act/tc_nat.h index c869274ac529..ae35f4009445 100644 --- a/include/net/tc_act/tc_nat.h +++ b/include/net/tc_act/tc_nat.h @@ -6,6 +6,7 @@ #include <net/act_api.h> struct tcf_nat_parms { + int action; __be32 old_addr; __be32 new_addr; __be32 mask; diff --git a/include/net/tc_act/tc_pedit.h b/include/net/tc_act/tc_pedit.h index 83fe39931781..f58ee15cd858 100644 --- a/include/net/tc_act/tc_pedit.h +++ b/include/net/tc_act/tc_pedit.h @@ -14,6 +14,7 @@ struct tcf_pedit_key_ex { struct tcf_pedit_parms { struct tc_pedit_key *tcfp_keys; struct tcf_pedit_key_ex *tcfp_keys_ex; + int action; u32 tcfp_off_max_hint; unsigned char tcfp_nkeys; unsigned char tcfp_flags; diff --git a/include/net/tc_act/tc_police.h b/include/net/tc_act/tc_police.h index 490d88cb5233..a89fc8e68b1e 100644 --- a/include/net/tc_act/tc_police.h +++ b/include/net/tc_act/tc_police.h @@ -5,10 +5,11 @@ #include <net/act_api.h> struct tcf_police_params { + int action; int tcfp_result; u32 tcfp_ewma_rate; - s64 tcfp_burst; u32 tcfp_mtu; + s64 tcfp_burst; s64 tcfp_mtu_ptoks; s64 tcfp_pkt_burst; struct psched_ratecfg rate; diff --git a/include/net/tc_act/tc_skbedit.h b/include/net/tc_act/tc_skbedit.h index 9649600fb3dc..31b2cd0bebb5 100644 --- a/include/net/tc_act/tc_skbedit.h +++ b/include/net/tc_act/tc_skbedit.h @@ -12,6 +12,7 @@ #include <linux/tc_act/tc_skbedit.h> struct tcf_skbedit_params { + int action; u32 flags; u32 priority; u32 mark; diff --git a/net/sched/act_connmark.c b/net/sched/act_connmark.c index 0fce631e7c91..3e89927d7116 100644 --- a/net/sched/act_connmark.c +++ b/net/sched/act_connmark.c @@ -88,7 +88,7 @@ count: /* using overlimits stats to count how many packets marked */ tcf_action_inc_overlimit_qstats(&ca->common); out: - return READ_ONCE(ca->tcf_action); + return parms->action; } static const struct nla_policy connmark_policy[TCA_CONNMARK_MAX + 1] = { @@ -167,6 +167,8 @@ static int tcf_connmark_init(struct net *net, struct nlattr *nla, if (err < 0) goto release_idr; + nparms->action = parm->action; + spin_lock_bh(&ci->tcf_lock); goto_ch = tcf_action_set_ctrlact(*a, parm->action, goto_ch); oparms = rcu_replace_pointer(ci->parms, nparms, lockdep_is_held(&ci->tcf_lock)); @@ -190,20 +192,20 @@ out_free: static inline int tcf_connmark_dump(struct sk_buff *skb, struct tc_action *a, int bind, int ref) { + const struct tcf_connmark_info *ci = to_connmark(a); unsigned char *b = skb_tail_pointer(skb); - struct tcf_connmark_info *ci = to_connmark(a); + const struct tcf_connmark_parms *parms; struct tc_connmark opt = { .index = ci->tcf_index, .refcnt = refcount_read(&ci->tcf_refcnt) - ref, .bindcnt = atomic_read(&ci->tcf_bindcnt) - bind, }; - struct tcf_connmark_parms *parms; struct tcf_t t; - spin_lock_bh(&ci->tcf_lock); - parms = rcu_dereference_protected(ci->parms, lockdep_is_held(&ci->tcf_lock)); + rcu_read_lock(); + parms = rcu_dereference(ci->parms); - opt.action = ci->tcf_action; + opt.action = parms->action; opt.zone = parms->zone; if (nla_put(skb, TCA_CONNMARK_PARMS, sizeof(opt), &opt)) goto nla_put_failure; @@ -212,12 +214,12 @@ static inline int tcf_connmark_dump(struct sk_buff *skb, struct tc_action *a, if (nla_put_64bit(skb, TCA_CONNMARK_TM, sizeof(t), &t, TCA_CONNMARK_PAD)) goto nla_put_failure; - spin_unlock_bh(&ci->tcf_lock); + rcu_read_unlock(); return skb->len; nla_put_failure: - spin_unlock_bh(&ci->tcf_lock); + rcu_read_unlock(); nlmsg_trim(skb, b); return -1; } diff --git a/net/sched/act_csum.c b/net/sched/act_csum.c index 5cc8e407e791..0939e6b2ba4d 100644 --- a/net/sched/act_csum.c +++ b/net/sched/act_csum.c @@ -99,6 +99,7 @@ static int tcf_csum_init(struct net *net, struct nlattr *nla, goto put_chain; } params_new->update_flags = parm->update_flags; + params_new->action = parm->action; spin_lock_bh(&p->tcf_lock); goto_ch = tcf_action_set_ctrlact(*a, parm->action, goto_ch); @@ -580,7 +581,7 @@ TC_INDIRECT_SCOPE int tcf_csum_act(struct sk_buff *skb, tcf_lastuse_update(&p->tcf_tm); tcf_action_update_bstats(&p->common, skb); - action = READ_ONCE(p->tcf_action); + action = params->action; if (unlikely(action == TC_ACT_SHOT)) goto drop; @@ -631,9 +632,9 @@ drop: static int tcf_csum_dump(struct sk_buff *skb, struct tc_action *a, int bind, int ref) { + const struct tcf_csum *p = to_tcf_csum(a); unsigned char *b = skb_tail_pointer(skb); - struct tcf_csum *p = to_tcf_csum(a); - struct tcf_csum_params *params; + const struct tcf_csum_params *params; struct tc_csum opt = { .index = p->tcf_index, .refcnt = refcount_read(&p->tcf_refcnt) - ref, @@ -641,10 +642,9 @@ static int tcf_csum_dump(struct sk_buff *skb, struct tc_action *a, int bind, }; struct tcf_t t; - spin_lock_bh(&p->tcf_lock); - params = rcu_dereference_protected(p->params, - lockdep_is_held(&p->tcf_lock)); - opt.action = p->tcf_action; + rcu_read_lock(); + params = rcu_dereference(p->params); + opt.action = params->action; opt.update_flags = params->update_flags; if (nla_put(skb, TCA_CSUM_PARMS, sizeof(opt), &opt)) @@ -653,12 +653,12 @@ static int tcf_csum_dump(struct sk_buff *skb, struct tc_action *a, int bind, tcf_tm_dump(&t, &p->tcf_tm); if (nla_put_64bit(skb, TCA_CSUM_TM, sizeof(t), &t, TCA_CSUM_PAD)) goto nla_put_failure; - spin_unlock_bh(&p->tcf_lock); + rcu_read_unlock(); return skb->len; nla_put_failure: - spin_unlock_bh(&p->tcf_lock); + rcu_read_unlock(); nlmsg_trim(skb, b); return -1; } diff --git a/net/sched/act_ct.c b/net/sched/act_ct.c index c02f39efc6ef..6749a4a9a9cd 100644 --- a/net/sched/act_ct.c +++ b/net/sched/act_ct.c @@ -977,7 +977,7 @@ TC_INDIRECT_SCOPE int tcf_ct_act(struct sk_buff *skb, const struct tc_action *a, p = rcu_dereference_bh(c->params); - retval = READ_ONCE(c->tcf_action); + retval = p->action; commit = p->ct_action & TCA_CT_ACT_COMMIT; clear = p->ct_action & TCA_CT_ACT_CLEAR; tmpl = p->tmpl; @@ -1409,6 +1409,7 @@ static int tcf_ct_init(struct net *net, struct nlattr *nla, if (err) goto cleanup; + params->action = parm->action; spin_lock_bh(&c->tcf_lock); goto_ch = tcf_action_set_ctrlact(*a, parm->action, goto_ch); params = rcu_replace_pointer(c->params, params, @@ -1442,8 +1443,8 @@ static void tcf_ct_cleanup(struct tc_action *a) } static int tcf_ct_dump_key_val(struct sk_buff *skb, - void *val, int val_type, - void *mask, int mask_type, + const void *val, int val_type, + const void *mask, int mask_type, int len) { int err; @@ -1464,9 +1465,9 @@ static int tcf_ct_dump_key_val(struct sk_buff *skb, return 0; } -static int tcf_ct_dump_nat(struct sk_buff *skb, struct tcf_ct_params *p) +static int tcf_ct_dump_nat(struct sk_buff *skb, const struct tcf_ct_params *p) { - struct nf_nat_range2 *range = &p->range; + const struct nf_nat_range2 *range = &p->range; if (!(p->ct_action & TCA_CT_ACT_NAT)) return 0; @@ -1504,7 +1505,8 @@ static int tcf_ct_dump_nat(struct sk_buff *skb, struct tcf_ct_params *p) return 0; } -static int tcf_ct_dump_helper(struct sk_buff *skb, struct nf_conntrack_helper *helper) +static int tcf_ct_dump_helper(struct sk_buff *skb, + const struct nf_conntrack_helper *helper) { if (!helper) return 0; @@ -1521,9 +1523,8 @@ static inline int tcf_ct_dump(struct sk_buff *skb, struct tc_action *a, int bind, int ref) { unsigned char *b = skb_tail_pointer(skb); - struct tcf_ct *c = to_ct(a); - struct tcf_ct_params *p; - + const struct tcf_ct *c = to_ct(a); + const struct tcf_ct_params *p; struct tc_ct opt = { .index = c->tcf_index, .refcnt = refcount_read(&c->tcf_refcnt) - ref, @@ -1531,10 +1532,9 @@ static inline int tcf_ct_dump(struct sk_buff *skb, struct tc_action *a, }; struct tcf_t t; - spin_lock_bh(&c->tcf_lock); - p = rcu_dereference_protected(c->params, - lockdep_is_held(&c->tcf_lock)); - opt.action = c->tcf_action; + rcu_read_lock(); + p = rcu_dereference(c->params); + opt.action = p->action; if (tcf_ct_dump_key_val(skb, &p->ct_action, TCA_CT_ACTION, @@ -1579,11 +1579,11 @@ skip_dump: tcf_tm_dump(&t, &c->tcf_tm); if (nla_put_64bit(skb, TCA_CT_TM, sizeof(t), &t, TCA_CT_PAD)) goto nla_put_failure; - spin_unlock_bh(&c->tcf_lock); + rcu_read_unlock(); return skb->len; nla_put_failure: - spin_unlock_bh(&c->tcf_lock); + rcu_read_unlock(); nlmsg_trim(skb, b); return -1; } diff --git a/net/sched/act_ctinfo.c b/net/sched/act_ctinfo.c index 5b1241ddc758..71efe04d00b5 100644 --- a/net/sched/act_ctinfo.c +++ b/net/sched/act_ctinfo.c @@ -44,9 +44,9 @@ static void tcf_ctinfo_dscp_set(struct nf_conn *ct, struct tcf_ctinfo *ca, ipv4_change_dsfield(ip_hdr(skb), INET_ECN_MASK, newdscp); - ca->stats_dscp_set++; + atomic64_inc(&ca->stats_dscp_set); } else { - ca->stats_dscp_error++; + atomic64_inc(&ca->stats_dscp_error); } } break; @@ -57,9 +57,9 @@ static void tcf_ctinfo_dscp_set(struct nf_conn *ct, struct tcf_ctinfo *ca, ipv6_change_dsfield(ipv6_hdr(skb), INET_ECN_MASK, newdscp); - ca->stats_dscp_set++; + atomic64_inc(&ca->stats_dscp_set); } else { - ca->stats_dscp_error++; + atomic64_inc(&ca->stats_dscp_error); } } break; @@ -72,7 +72,7 @@ static void tcf_ctinfo_cpmark_set(struct nf_conn *ct, struct tcf_ctinfo *ca, struct tcf_ctinfo_params *cp, struct sk_buff *skb) { - ca->stats_cpmark_set++; + atomic64_inc(&ca->stats_cpmark_set); skb->mark = READ_ONCE(ct->mark) & cp->cpmarkmask; } @@ -88,13 +88,11 @@ TC_INDIRECT_SCOPE int tcf_ctinfo_act(struct sk_buff *skb, struct tcf_ctinfo_params *cp; struct nf_conn *ct; int proto, wlen; - int action; cp = rcu_dereference_bh(ca->params); tcf_lastuse_update(&ca->tcf_tm); tcf_action_update_bstats(&ca->common, skb); - action = READ_ONCE(ca->tcf_action); wlen = skb_network_offset(skb); switch (skb_protocol(skb, true)) { @@ -141,7 +139,7 @@ TC_INDIRECT_SCOPE int tcf_ctinfo_act(struct sk_buff *skb, if (thash) nf_ct_put(ct); out: - return action; + return cp->action; } static const struct nla_policy ctinfo_policy[TCA_CTINFO_MAX + 1] = { @@ -258,6 +256,8 @@ static int tcf_ctinfo_init(struct net *net, struct nlattr *nla, cp_new->mode |= CTINFO_MODE_CPMARK; } + cp_new->action = actparm->action; + spin_lock_bh(&ci->tcf_lock); goto_ch = tcf_action_set_ctrlact(*a, actparm->action, goto_ch); cp_new = rcu_replace_pointer(ci->params, cp_new, @@ -282,25 +282,24 @@ release_idr: static int tcf_ctinfo_dump(struct sk_buff *skb, struct tc_action *a, int bind, int ref) { - struct tcf_ctinfo *ci = to_ctinfo(a); + const struct tcf_ctinfo *ci = to_ctinfo(a); + unsigned char *b = skb_tail_pointer(skb); + const struct tcf_ctinfo_params *cp; struct tc_ctinfo opt = { .index = ci->tcf_index, .refcnt = refcount_read(&ci->tcf_refcnt) - ref, .bindcnt = atomic_read(&ci->tcf_bindcnt) - bind, }; - unsigned char *b = skb_tail_pointer(skb); - struct tcf_ctinfo_params *cp; struct tcf_t t; - spin_lock_bh(&ci->tcf_lock); - cp = rcu_dereference_protected(ci->params, - lockdep_is_held(&ci->tcf_lock)); + rcu_read_lock(); + cp = rcu_dereference(ci->params); tcf_tm_dump(&t, &ci->tcf_tm); if (nla_put_64bit(skb, TCA_CTINFO_TM, sizeof(t), &t, TCA_CTINFO_PAD)) goto nla_put_failure; - opt.action = ci->tcf_action; + opt.action = cp->action; if (nla_put(skb, TCA_CTINFO_ACT, sizeof(opt), &opt)) goto nla_put_failure; @@ -323,22 +322,25 @@ static int tcf_ctinfo_dump(struct sk_buff *skb, struct tc_action *a, } if (nla_put_u64_64bit(skb, TCA_CTINFO_STATS_DSCP_SET, - ci->stats_dscp_set, TCA_CTINFO_PAD)) + atomic64_read(&ci->stats_dscp_set), + TCA_CTINFO_PAD)) goto nla_put_failure; if (nla_put_u64_64bit(skb, TCA_CTINFO_STATS_DSCP_ERROR, - ci->stats_dscp_error, TCA_CTINFO_PAD)) + atomic64_read(&ci->stats_dscp_error), + TCA_CTINFO_PAD)) goto nla_put_failure; if (nla_put_u64_64bit(skb, TCA_CTINFO_STATS_CPMARK_SET, - ci->stats_cpmark_set, TCA_CTINFO_PAD)) + atomic64_read(&ci->stats_cpmark_set), + TCA_CTINFO_PAD)) goto nla_put_failure; - spin_unlock_bh(&ci->tcf_lock); + rcu_read_unlock(); return skb->len; nla_put_failure: - spin_unlock_bh(&ci->tcf_lock); + rcu_read_unlock(); nlmsg_trim(skb, b); return -1; } diff --git a/net/sched/act_mpls.c b/net/sched/act_mpls.c index 9f86f4e666d3..6654011dcd2b 100644 --- a/net/sched/act_mpls.c +++ b/net/sched/act_mpls.c @@ -57,7 +57,7 @@ TC_INDIRECT_SCOPE int tcf_mpls_act(struct sk_buff *skb, struct tcf_mpls *m = to_mpls(a); struct tcf_mpls_params *p; __be32 new_lse; - int ret, mac_len; + int mac_len; tcf_lastuse_update(&m->tcf_tm); bstats_update(this_cpu_ptr(m->common.cpu_bstats), skb); @@ -72,8 +72,6 @@ TC_INDIRECT_SCOPE int tcf_mpls_act(struct sk_buff *skb, mac_len = skb_network_offset(skb); } - ret = READ_ONCE(m->tcf_action); - p = rcu_dereference_bh(m->mpls_p); switch (p->tcfm_action) { @@ -122,7 +120,7 @@ TC_INDIRECT_SCOPE int tcf_mpls_act(struct sk_buff *skb, if (skb_at_tc_ingress(skb)) skb_pull_rcsum(skb, skb->mac_len); - return ret; + return p->action; drop: qstats_drop_inc(this_cpu_ptr(m->common.cpu_qstats)); @@ -296,6 +294,7 @@ static int tcf_mpls_init(struct net *net, struct nlattr *nla, ACT_MPLS_BOS_NOT_SET); p->tcfm_proto = nla_get_be16_default(tb[TCA_MPLS_PROTO], htons(ETH_P_MPLS_UC)); + p->action = parm->action; spin_lock_bh(&m->tcf_lock); goto_ch = tcf_action_set_ctrlact(*a, parm->action, goto_ch); @@ -330,8 +329,8 @@ static int tcf_mpls_dump(struct sk_buff *skb, struct tc_action *a, int bind, int ref) { unsigned char *b = skb_tail_pointer(skb); - struct tcf_mpls *m = to_mpls(a); - struct tcf_mpls_params *p; + const struct tcf_mpls *m = to_mpls(a); + const struct tcf_mpls_params *p; struct tc_mpls opt = { .index = m->tcf_index, .refcnt = refcount_read(&m->tcf_refcnt) - ref, @@ -339,10 +338,10 @@ static int tcf_mpls_dump(struct sk_buff *skb, struct tc_action *a, }; struct tcf_t t; - spin_lock_bh(&m->tcf_lock); - opt.action = m->tcf_action; - p = rcu_dereference_protected(m->mpls_p, lockdep_is_held(&m->tcf_lock)); + rcu_read_lock(); + p = rcu_dereference(m->mpls_p); opt.m_action = p->tcfm_action; + opt.action = p->action; if (nla_put(skb, TCA_MPLS_PARMS, sizeof(opt), &opt)) goto nla_put_failure; @@ -370,12 +369,12 @@ static int tcf_mpls_dump(struct sk_buff *skb, struct tc_action *a, if (nla_put_64bit(skb, TCA_MPLS_TM, sizeof(t), &t, TCA_MPLS_PAD)) goto nla_put_failure; - spin_unlock_bh(&m->tcf_lock); + rcu_read_unlock(); return skb->len; nla_put_failure: - spin_unlock_bh(&m->tcf_lock); + rcu_read_unlock(); nlmsg_trim(skb, b); return -EMSGSIZE; } diff --git a/net/sched/act_nat.c b/net/sched/act_nat.c index d541f553805f..26241d80ebe0 100644 --- a/net/sched/act_nat.c +++ b/net/sched/act_nat.c @@ -91,6 +91,7 @@ static int tcf_nat_init(struct net *net, struct nlattr *nla, struct nlattr *est, nparm->new_addr = parm->new_addr; nparm->mask = parm->mask; nparm->flags = parm->flags; + nparm->action = parm->action; p = to_tcf_nat(*a); @@ -130,17 +131,16 @@ TC_INDIRECT_SCOPE int tcf_nat_act(struct sk_buff *skb, tcf_lastuse_update(&p->tcf_tm); tcf_action_update_bstats(&p->common, skb); - action = READ_ONCE(p->tcf_action); - parms = rcu_dereference_bh(p->parms); + action = parms->action; + if (unlikely(action == TC_ACT_SHOT)) + goto drop; + old_addr = parms->old_addr; new_addr = parms->new_addr; mask = parms->mask; egress = parms->flags & TCA_NAT_FLAG_EGRESS; - if (unlikely(action == TC_ACT_SHOT)) - goto drop; - noff = skb_network_offset(skb); if (!pskb_may_pull(skb, sizeof(*iph) + noff)) goto drop; @@ -268,21 +268,20 @@ static int tcf_nat_dump(struct sk_buff *skb, struct tc_action *a, int bind, int ref) { unsigned char *b = skb_tail_pointer(skb); - struct tcf_nat *p = to_tcf_nat(a); + const struct tcf_nat *p = to_tcf_nat(a); + const struct tcf_nat_parms *parms; struct tc_nat opt = { .index = p->tcf_index, .refcnt = refcount_read(&p->tcf_refcnt) - ref, .bindcnt = atomic_read(&p->tcf_bindcnt) - bind, }; - struct tcf_nat_parms *parms; struct tcf_t t; - spin_lock_bh(&p->tcf_lock); - - opt.action = p->tcf_action; + rcu_read_lock(); - parms = rcu_dereference_protected(p->parms, lockdep_is_held(&p->tcf_lock)); + parms = rcu_dereference(p->parms); + opt.action = parms->action; opt.old_addr = parms->old_addr; opt.new_addr = parms->new_addr; opt.mask = parms->mask; @@ -294,12 +293,12 @@ static int tcf_nat_dump(struct sk_buff *skb, struct tc_action *a, tcf_tm_dump(&t, &p->tcf_tm); if (nla_put_64bit(skb, TCA_NAT_TM, sizeof(t), &t, TCA_NAT_PAD)) goto nla_put_failure; - spin_unlock_bh(&p->tcf_lock); + rcu_read_unlock(); return skb->len; nla_put_failure: - spin_unlock_bh(&p->tcf_lock); + rcu_read_unlock(); nlmsg_trim(skb, b); return -1; } diff --git a/net/sched/act_pedit.c b/net/sched/act_pedit.c index fc0a35a7b62a..4b65901397a8 100644 --- a/net/sched/act_pedit.c +++ b/net/sched/act_pedit.c @@ -279,7 +279,7 @@ static int tcf_pedit_init(struct net *net, struct nlattr *nla, } p = to_pedit(*a); - + nparms->action = parm->action; spin_lock_bh(&p->tcf_lock); goto_ch = tcf_action_set_ctrlact(*a, parm->action, goto_ch); oparms = rcu_replace_pointer(p->parms, nparms, 1); @@ -483,7 +483,7 @@ TC_INDIRECT_SCOPE int tcf_pedit_act(struct sk_buff *skb, bad: tcf_action_inc_overlimit_qstats(&p->common); done: - return p->tcf_action; + return parms->action; } static void tcf_pedit_stats_update(struct tc_action *a, u64 bytes, u64 packets, @@ -500,19 +500,19 @@ static int tcf_pedit_dump(struct sk_buff *skb, struct tc_action *a, int bind, int ref) { unsigned char *b = skb_tail_pointer(skb); - struct tcf_pedit *p = to_pedit(a); - struct tcf_pedit_parms *parms; + const struct tcf_pedit *p = to_pedit(a); + const struct tcf_pedit_parms *parms; struct tc_pedit *opt; struct tcf_t t; int s; - spin_lock_bh(&p->tcf_lock); - parms = rcu_dereference_protected(p->parms, 1); + rcu_read_lock(); + parms = rcu_dereference(p->parms); s = struct_size(opt, keys, parms->tcfp_nkeys); opt = kzalloc(s, GFP_ATOMIC); if (unlikely(!opt)) { - spin_unlock_bh(&p->tcf_lock); + rcu_read_unlock(); return -ENOBUFS; } opt->nkeys = parms->tcfp_nkeys; @@ -521,7 +521,7 @@ static int tcf_pedit_dump(struct sk_buff *skb, struct tc_action *a, flex_array_size(opt, keys, parms->tcfp_nkeys)); opt->index = p->tcf_index; opt->flags = parms->tcfp_flags; - opt->action = p->tcf_action; + opt->action = parms->action; opt->refcnt = refcount_read(&p->tcf_refcnt) - ref; opt->bindcnt = atomic_read(&p->tcf_bindcnt) - bind; @@ -540,13 +540,13 @@ static int tcf_pedit_dump(struct sk_buff *skb, struct tc_action *a, tcf_tm_dump(&t, &p->tcf_tm); if (nla_put_64bit(skb, TCA_PEDIT_TM, sizeof(t), &t, TCA_PEDIT_PAD)) goto nla_put_failure; - spin_unlock_bh(&p->tcf_lock); + rcu_read_unlock(); kfree(opt); return skb->len; nla_put_failure: - spin_unlock_bh(&p->tcf_lock); + rcu_read_unlock(); nlmsg_trim(skb, b); kfree(opt); return -1; diff --git a/net/sched/act_police.c b/net/sched/act_police.c index a214ed681142..0e1c61183379 100644 --- a/net/sched/act_police.c +++ b/net/sched/act_police.c @@ -198,6 +198,7 @@ static int tcf_police_init(struct net *net, struct nlattr *nla, psched_ppscfg_precompute(&new->ppsrate, pps); } + new->action = parm->action; spin_lock_bh(&police->tcf_lock); spin_lock_bh(&police->tcfp_lock); police->tcfp_t_c = ktime_get_ns(); @@ -254,8 +255,8 @@ TC_INDIRECT_SCOPE int tcf_police_act(struct sk_buff *skb, tcf_lastuse_update(&police->tcf_tm); bstats_update(this_cpu_ptr(police->common.cpu_bstats), skb); - ret = READ_ONCE(police->tcf_action); p = rcu_dereference_bh(police->params); + ret = p->action; if (p->tcfp_ewma_rate) { struct gnet_stats_rate_est64 sample; @@ -338,9 +339,9 @@ static void tcf_police_stats_update(struct tc_action *a, static int tcf_police_dump(struct sk_buff *skb, struct tc_action *a, int bind, int ref) { + const struct tcf_police *police = to_police(a); unsigned char *b = skb_tail_pointer(skb); - struct tcf_police *police = to_police(a); - struct tcf_police_params *p; + const struct tcf_police_params *p; struct tc_police opt = { .index = police->tcf_index, .refcnt = refcount_read(&police->tcf_refcnt) - ref, @@ -348,10 +349,9 @@ static int tcf_police_dump(struct sk_buff *skb, struct tc_action *a, }; struct tcf_t t; - spin_lock_bh(&police->tcf_lock); - opt.action = police->tcf_action; - p = rcu_dereference_protected(police->params, - lockdep_is_held(&police->tcf_lock)); + rcu_read_lock(); + p = rcu_dereference(police->params); + opt.action = p->action; opt.mtu = p->tcfp_mtu; opt.burst = PSCHED_NS2TICKS(p->tcfp_burst); if (p->rate_present) { @@ -392,12 +392,12 @@ static int tcf_police_dump(struct sk_buff *skb, struct tc_action *a, tcf_tm_dump(&t, &police->tcf_tm); if (nla_put_64bit(skb, TCA_POLICE_TM, sizeof(t), &t, TCA_POLICE_PAD)) goto nla_put_failure; - spin_unlock_bh(&police->tcf_lock); + rcu_read_unlock(); return skb->len; nla_put_failure: - spin_unlock_bh(&police->tcf_lock); + rcu_read_unlock(); nlmsg_trim(skb, b); return -1; } diff --git a/net/sched/act_skbedit.c b/net/sched/act_skbedit.c index 1f1d9ce3e968..8c1d1554f657 100644 --- a/net/sched/act_skbedit.c +++ b/net/sched/act_skbedit.c @@ -43,13 +43,11 @@ TC_INDIRECT_SCOPE int tcf_skbedit_act(struct sk_buff *skb, { struct tcf_skbedit *d = to_skbedit(a); struct tcf_skbedit_params *params; - int action; tcf_lastuse_update(&d->tcf_tm); bstats_update(this_cpu_ptr(d->common.cpu_bstats), skb); params = rcu_dereference_bh(d->params); - action = READ_ONCE(d->tcf_action); if (params->flags & SKBEDIT_F_PRIORITY) skb->priority = params->priority; @@ -85,7 +83,7 @@ TC_INDIRECT_SCOPE int tcf_skbedit_act(struct sk_buff *skb, } if (params->flags & SKBEDIT_F_PTYPE) skb->pkt_type = params->ptype; - return action; + return params->action; err: qstats_drop_inc(this_cpu_ptr(d->common.cpu_qstats)); @@ -262,6 +260,7 @@ static int tcf_skbedit_init(struct net *net, struct nlattr *nla, if (flags & SKBEDIT_F_MASK) params_new->mask = *mask; + params_new->action = parm->action; spin_lock_bh(&d->tcf_lock); goto_ch = tcf_action_set_ctrlact(*a, parm->action, goto_ch); params_new = rcu_replace_pointer(d->params, params_new, @@ -284,9 +283,9 @@ release_idr: static int tcf_skbedit_dump(struct sk_buff *skb, struct tc_action *a, int bind, int ref) { + const struct tcf_skbedit *d = to_skbedit(a); unsigned char *b = skb_tail_pointer(skb); - struct tcf_skbedit *d = to_skbedit(a); - struct tcf_skbedit_params *params; + const struct tcf_skbedit_params *params; struct tc_skbedit opt = { .index = d->tcf_index, .refcnt = refcount_read(&d->tcf_refcnt) - ref, @@ -295,10 +294,9 @@ static int tcf_skbedit_dump(struct sk_buff *skb, struct tc_action *a, u64 pure_flags = 0; struct tcf_t t; - spin_lock_bh(&d->tcf_lock); - params = rcu_dereference_protected(d->params, - lockdep_is_held(&d->tcf_lock)); - opt.action = d->tcf_action; + rcu_read_lock(); + params = rcu_dereference(d->params); + opt.action = params->action; if (nla_put(skb, TCA_SKBEDIT_PARMS, sizeof(opt), &opt)) goto nla_put_failure; @@ -333,12 +331,12 @@ static int tcf_skbedit_dump(struct sk_buff *skb, struct tc_action *a, tcf_tm_dump(&t, &d->tcf_tm); if (nla_put_64bit(skb, TCA_SKBEDIT_TM, sizeof(t), &t, TCA_SKBEDIT_PAD)) goto nla_put_failure; - spin_unlock_bh(&d->tcf_lock); + rcu_read_unlock(); return skb->len; nla_put_failure: - spin_unlock_bh(&d->tcf_lock); + rcu_read_unlock(); nlmsg_trim(skb, b); return -1; } |