<div dir="ltr"><br><br><div class="gmail_quote"><div dir="ltr" class="gmail_attr">---------- Forwarded message ---------<br>From: <strong class="gmail_sendername" dir="auto">Paolo Abeni</strong> <span dir="auto"><<a href="mailto:pabeni@redhat.com">pabeni@redhat.com</a>></span><br>Date: Wed, May 8, 2024 at 1:27 PM<br>Subject: [RFC PATCH] net: introduce HW Rate Limiting Driver API<br>To: <<a href="mailto:netdev@vger.kernel.org">netdev@vger.kernel.org</a>><br>Cc: Jakub Kicinski <<a href="mailto:kuba@kernel.org">kuba@kernel.org</a>>, Jiri Pirko <<a href="mailto:jiri@resnulli.us">jiri@resnulli.us</a>>, Madhu Chittim <<a href="mailto:madhu.chittim@intel.com">madhu.chittim@intel.com</a>>, Sridhar Samudrala <<a href="mailto:sridhar.samudrala@intel.com">sridhar.samudrala@intel.com</a>>, Simon Horman <<a href="mailto:horms@kernel.org">horms@kernel.org</a>>, John Fastabend <<a href="mailto:john.fastabend@gmail.com">john.fastabend@gmail.com</a>>, Sunil Kovvuri Goutham <<a href="mailto:sgoutham@marvell.com">sgoutham@marvell.com</a>>, Jamal Hadi Salim <<a href="mailto:jhs@mojatatu.com">jhs@mojatatu.com</a>><br></div><br><br>This is the first incarnation in a formal (pre-RFC) patch of the<br>
HW TX Rate Limiting Driver API proposal shared here[1].<br>
<br>
The goal is to outline the proposed APIs before pushing the actual<br>
implementation.<br>
<br>
The network devices gain a new ops struct to directly manipulate the<br>
H/W shapers implemented by the NIC.<br>
<br>
The shapers can be attached to a pre-defined set of 'domains' - port,<br>
vf, etc. - and the overall shapers configuration pushed to the H/W is<br>
maintained by the kernel.<br>
<br>
Each shaper is identified by an unique integer id based on the domain<br>
and additional domain-specific information - e.g. for the VF domain, the<br>
virtual function number/identifier.<br>
<br>
[1] <a href="https://lore.kernel.org/netdev/20240405102313.GA310894@kernel.org/" rel="noreferrer" target="_blank">https://lore.kernel.org/netdev/20240405102313.GA310894@kernel.org/</a><br>
<br>
Co-developed-by: Simon Horman <<a href="mailto:horms@kernel.org" target="_blank">horms@kernel.org</a>><br>
Signed-off-by: Simon Horman <<a href="mailto:horms@kernel.org" target="_blank">horms@kernel.org</a>><br>
Signed-off-by: Paolo Abeni <<a href="mailto:pabeni@redhat.com" target="_blank">pabeni@redhat.com</a>><br>
---<br>
include/linux/netdevice.h | 15 +++<br>
include/net/net_shaper.h | 206 ++++++++++++++++++++++++++++++++++++++<br>
net/Kconfig | 3 +<br>
3 files changed, 224 insertions(+)<br>
create mode 100644 include/net/net_shaper.h<br>
<br>
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h<br>
index cf261fb89d73..39f66af014be 100644<br>
--- a/include/linux/netdevice.h<br>
+++ b/include/linux/netdevice.h<br>
@@ -79,6 +79,8 @@ struct xdp_buff;<br>
struct xdp_frame;<br>
struct xdp_metadata_ops;<br>
struct xdp_md;<br>
+struct net_shaper_ops;<br>
+struct net_shaper_data;<br>
<br>
typedef u32 xdp_features_t;<br>
<br>
@@ -1596,6 +1598,13 @@ struct net_device_ops {<br>
int (*ndo_hwtstamp_set)(struct net_device *dev,<br>
struct kernel_hwtstamp_config *kernel_config,<br>
struct netlink_ext_ack *extack);<br>
+<br>
+#if IS_ENABLED(CONFIG_NET_SHAPER)<br>
+ /** @net_shaper_ops: Device shaping offload operations<br>
+ * see include/net/net_shapers.h<br>
+ */<br>
+ const struct net_shaper_ops *net_shaper_ops;<br>
+#endif<br>
};<br>
<br>
/**<br>
@@ -2403,6 +2412,12 @@ struct net_device {<br>
/** @page_pools: page pools created for this netdevice */<br>
struct hlist_head page_pools;<br>
#endif<br>
+#if IS_ENABLED(CONFIG_NET_SHAPER)<br>
+ /** @net_shaper_data: data tracking the current shaper status<br>
+ * see include/net/net_shapers.h<br>
+ */<br>
+ struct net_shaper_data *net_shaper_data;<br>
+#endif<br>
};<br>
#define to_net_dev(d) container_of(d, struct net_device, dev)<br>
<br>
diff --git a/include/net/net_shaper.h b/include/net/net_shaper.h<br>
new file mode 100644<br>
index 000000000000..a4fbadd99870<br>
--- /dev/null<br>
+++ b/include/net/net_shaper.h<br>
@@ -0,0 +1,206 @@<br>
+/* SPDX-License-Identifier: GPL-2.0-or-later */<br>
+<br>
+#ifndef _NET_SHAPER_H_<br>
+#define _NET_SHAPER_H_<br>
+<br>
+#include <linux/types.h><br>
+#include <linux/netdevice.h><br>
+#include <linux/netlink.h><br>
+<br>
+/**<br>
+ * enum net_shaper_metric - the metric of the shaper<br>
+ * @NET_SHAPER_METRIC_PPS: Shaper operates on a packets per second basis<br>
+ * @NET_SHAPER_METRIC_BPS: Shaper operates on a bits per second basis<br>
+ */<br>
+enum net_shaper_metric {<br>
+ NET_SHAPER_METRIC_PPS,<br>
+ NET_SHAPER_METRIC_BPS<br>
+};<br>
+<br>
+/**<br>
+ * struct net_shaper_info - represents a shaping node on the NIC H/W<br>
+ * @metric: Specify if the bw limits refers to PPS or BPS<br>
+ * @bw_min: Minimum guaranteed rate for this shaper<br>
+ * @bw_max: Maximum peak bw allowed for this shaper<br>
+ * @burst: Maximum burst for the peek rate of this shaper<br>
+ * @priority: Scheduling priority for this shaper<br>
+ * @weight: Scheduling weight for this shaper<br>
+ */<br>
+struct net_shaper_info {<br>
+ enum net_shaper_metric metric;<br>
+ u64 bw_min; /* minimum guaranteed bandwidth, according to metric */<br>
+ u64 bw_max; /* maximum allowed bandwidth */<br>
+ u32 burst; /* maximum burst in bytes for bw_max */<br>
+ u32 priority; /* scheduling strict priority */<br>
+ u32 weight; /* scheduling WRR weight*/<br>
+};<br>
+<br>
+/**<br>
+ * enum net_shaper_scope - the different scopes where a shaper could be attached<br>
+ * @NET_SHAPER_SCOPE_PORT: The root shaper for the whole H/W.<br>
+ * @NET_SHAPER_SCOPE_NETDEV: The main shaper for the given network device.<br>
+ * @NET_SHAPER_SCOPE_VF: The shaper is attached to the given virtual<br>
+ * function.<br>
+ * @NET_SHAPER_SCOPE_QUEUE_GROUP: The shaper groups multiple queues under the<br>
+ * same device.<br>
+ * @NET_SHAPER_SCOPE_QUEUE: The shaper is attached to the given device queue.<br>
+ *<br>
+ * NET_SHAPER_SCOPE_PORT and NET_SHAPER_SCOPE_VF are only available on<br>
+ * PF devices, usually inside the host/hypervisor.<br>
+ * NET_SHAPER_SCOPE_NETDEV, NET_SHAPER_SCOPE_QUEUE_GROUP and<br>
+ * NET_SHAPER_SCOPE_QUEUE are available on both PFs and VFs devices.<br>
+ */<br>
+enum net_shaper_scope {<br>
+ NET_SHAPER_SCOPE_PORT,<br>
+ NET_SHAPER_SCOPE_NETDEV,<br>
+ NET_SHAPER_SCOPE_VF,<br>
+ NET_SHAPER_SCOPE_QUEUE_GROUP,<br>
+ NET_SHAPER_SCOPE_QUEUE,<br>
+};<br>
+<br>
+/**<br>
+ * struct net_shaper_ops - Operations on device H/W shapers<br>
+ * @add: Creates a new shaper in the specified scope.<br>
+ * @set: Modify the existing shaper.<br>
+ * @delete: Delete the specified shaper.<br>
+ * @move: Move an existing shaper under a different parent.<br>
+ *<br>
+ * The initial shaping configuration ad device initialization is empty/<br>
+ * a no-op/does not constraint the b/w in any way.<br>
+ * The network core keeps track of the applied user-configuration in<br>
+ * per device storage.<br>
+ *<br>
+ * Each shaper is uniquely identified within the device with an 'handle',<br>
+ * dependent on the shaper scope and other data, see @shaper_make_handle()<br>
+ */<br>
+struct net_shaper_ops {<br>
+ /** add - Add a shaper inside the shaper hierarchy<br>
+ * @dev: netdevice to operate on<br>
+ * @handle: the shaper indetifier<br>
+ * @shaper: configuration of shaper<br>
+ * @extack: Netlink extended ACK for reporting errors.<br>
+ *<br>
+ * Return:<br>
+ * * 0 on success<br>
+ * * %-EOPNOTSUPP - Operation is not supported by hardware, driver,<br>
+ * or core for any reason. @extack should be set to<br>
+ * text describing the reason.<br>
+ * * Other negative error values on failure.<br>
+ *<br>
+ * Examples or reasons this operation may fail include:<br>
+ * * H/W resources limits.<br>
+ * * Can’t respect the requested bw limits.<br>
+ */<br>
+ int (*add)(struct net_device *dev, u32 handle,<br>
+ const struct net_shaper_info *shaper,<br>
+ struct netlink_ext_ack *extack);<br>
+<br>
+ /** set - Update the specified shaper, if it exists<br>
+ * @dev: Netdevice to operate on.<br>
+ * @handle: the shaper identifier<br>
+ * @shaper: Configuration of shaper.<br>
+ * @extack: Netlink extended ACK for reporting errors.<br>
+ *<br>
+ * Return:<br>
+ * * %0 - Success<br>
+ * * %-EOPNOTSUPP - Operation is not supported by hardware, driver,<br>
+ * or core for any reason. @extack should be set to<br>
+ * text describing the reason.<br>
+ * * Other negative error values on failure.<br>
+ */<br>
+ int (*set)(struct net_device *dev, u32 handle,<br>
+ const struct net_shaper_info *shaper,<br>
+ struct netlink_ext_ack *extack);<br>
+<br>
+ /** delete - Removes a shaper from the NIC<br>
+ * @dev: netdevice to operate on.<br>
+ * @handle: the shaper identifier<br>
+ * @extack: Netlink extended ACK for reporting errors.<br>
+ *<br>
+ * Return:<br>
+ * * %0 - Success<br>
+ * * %-EOPNOTSUPP - Operation is not supported by hardware, driver,<br>
+ * or core for any reason. @extack should be set to<br>
+ * text describing the reason.<br>
+ * * Other negative error value on failure.<br>
+ */<br>
+ int (*delete)(struct net_device *dev, u32 handle,<br>
+ struct netlink_ext_ack *extack);<br>
+<br>
+ /** Move - change the parent id of the specified shaper<br>
+ * @dev: netdevice to operate on.<br>
+ * @handle: unique identifier for the shaper<br>
+ * @new_parent_id: identifier of the new parent for this shaper<br>
+ * @extack: Netlink extended ACK for reporting errors.<br>
+ *<br>
+ * Move the specified shaper in the hierarchy replacing its<br>
+ * current parent shaper with @new_parent_id<br>
+ *<br>
+ * Return:<br>
+ * * %0 - Success<br>
+ * * %-EOPNOTSUPP - Operation is not supported by hardware, driver,<br>
+ * or core for any reason. @extack should be set to<br>
+ * text describing the reason.<br>
+ * * Other negative error values on failure.<br>
+ */<br>
+ int (*move)(struct net_device *dev, u32 handle,<br>
+ u32 new_parent_handle, struct netlink_ext_ack *extack);<br>
+};<br>
+<br>
+/**<br>
+ * net_shaper_make_handle - creates an unique shaper identifier<br>
+ * @scope: the shaper scope<br>
+ * @vf: virtual function number<br>
+ * @id: queue group or queue id<br>
+ *<br>
+ * Return: an unique identifier for the shaper<br>
+ *<br>
+ * Combines the specified arguments to create an unique identifier for<br>
+ * the shaper.<br>
+ * The virtual function number is only used within @NET_SHAPER_SCOPE_VF,<br>
+ * @NET_SHAPER_SCOPE_QUEUE_GROUP and @NET_SHAPER_SCOPE_QUEUE.<br>
+ * The @id number is only used for @NET_SHAPER_SCOPE_QUEUE_GROUP and<br>
+ * @NET_SHAPER_SCOPE_QUEUE, and must be, respectively, the queue group<br>
+ * identifier or the queue number.<br>
+ */<br>
+u32 net_shaper_make_handle(enum net_shaper_scope scope, int vf, int id);<br>
+<br>
+/*<br>
+ * Examples:<br>
+ * - set shaping on a given queue<br>
+ * struct shaper_info info = { }; // fill this<br>
+ * u32 handle = shaper_make_handle(NET_SHAPER_SCOPE_QUEUE, 0, queue_id);<br>
+ * dev->shaper_ops->add(dev, handle, &info, NULL);<br>
+ *<br>
+ * - create a queue group with a queue group shaping limits.<br>
+ * Assuming the following topology already exists:<br>
+ * < netdev shaper ><br>
+ * / \<br>
+ * <queue 0 shaper> . . . <queue N shaper><br>
+ *<br>
+ * struct shaper_info ginfo = { }; // fill this<br>
+ * u32 ghandle = shaper_make_handle(NET_SHAPER_SCOPE_QUEUE_GROUP, 0, 0);<br>
+ * dev->shaper_ops->add(dev, ghandle, &ginfo);<br>
+ *<br>
+ * // now topology is:<br>
+ * // < netdev shaper ><br>
+ * // / | \<br>
+ * // / | < newly created shaper ><br>
+ * // / |<br>
+ * // <queue 0 shaper> . . . <queue N shaper><br>
+ *<br>
+ * // move a shapers for queues 3..n out of such queue group<br>
+ * for (i = 0; i <= 2; ++i) {<br>
+ * u32 qhandle = net_shaper_make_handle(NET_SHAPER_SCOPE_QUEUE, 0, i);<br>
+ * dev->netshaper_ops->move(dev, qhandle, ghandle, NULL);<br>
+ * }<br>
+ *<br>
+ * // now the topology is:<br>
+ * // < netdev shaper ><br>
+ * // / \<br>
+ * // < newly created shaper> <queue 3 shaper> .. <queue n shaper><br>
+ * // / \<br>
+ * // <queue 0 shaper> . . . <queue 2 shaper><br>
+ */<br>
+#endif<br>
+<br>
diff --git a/net/Kconfig b/net/Kconfig<br>
index f0a8692496ff..29c6fec54711 100644<br>
--- a/net/Kconfig<br>
+++ b/net/Kconfig<br>
@@ -66,6 +66,9 @@ config SKB_DECRYPTED<br>
config SKB_EXTENSIONS<br>
bool<br>
<br>
+config NET_SHAPER<br>
+ bool<br>
+<br>
menu "Networking options"<br>
<br>
source "net/packet/Kconfig"<br>
-- <br>
2.43.2<br>
<br>
<br>
</div><br clear="all"><div><br></div><span class="gmail_signature_prefix">-- </span><br><div dir="ltr" class="gmail_signature" data-smartmail="gmail_signature"><div dir="ltr"><div><a href="https://www.linkedin.com/feed/update/urn:li:activity:7203400057172180992/" target="_blank">https://www.linkedin.com/feed/update/urn:li:activity:7203400057172180992/</a></div><div>Donations Drive.</div><div>Dave Täht CSO, LibreQos<br></div></div></div></div>