static struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
static struct list_head ptype_all __read_mostly; /* Taps */
-#ifdef CONFIG_NET_DMA
-struct net_dma {
- struct dma_client client;
- spinlock_t lock;
- cpumask_t channel_mask;
- struct dma_chan **channels;
-};
-
-static enum dma_state_client
-netdev_dma_event(struct dma_client *client, struct dma_chan *chan,
- enum dma_state state);
-
-static struct net_dma net_dma = {
- .client = {
- .event_callback = netdev_dma_event,
- },
-};
-#endif
-
/*
* The @dev_base_head list is protected by @dev_base_lock and the rtnl
* semaphore.
}
EXPORT_SYMBOL(napi_gro_flush);
-static int __napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
+int dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
{
struct sk_buff **pp = NULL;
struct packet_type *ptype;
for (p = napi->gro_list; p; p = p->next) {
count++;
- NAPI_GRO_CB(p)->same_flow =
- p->mac_len == mac_len &&
- !memcmp(skb_mac_header(p), skb_mac_header(skb),
- mac_len);
- NAPI_GRO_CB(p)->flush = 0;
+
+ if (!NAPI_GRO_CB(p)->same_flow)
+ continue;
+
+ if (p->mac_len != mac_len ||
+ memcmp(skb_mac_header(p), skb_mac_header(skb),
+ mac_len))
+ NAPI_GRO_CB(p)->same_flow = 0;
}
pp = ptype->gro_receive(&napi->gro_list, skb);
normal:
return -1;
}
+EXPORT_SYMBOL(dev_gro_receive);
+
+static int __napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
+{
+ struct sk_buff *p;
+
+ for (p = napi->gro_list; p; p = p->next) {
+ NAPI_GRO_CB(p)->same_flow = 1;
+ NAPI_GRO_CB(p)->flush = 0;
+ }
+
+ return dev_gro_receive(napi, skb);
+}
int napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
{
}
EXPORT_SYMBOL(napi_gro_receive);
-int napi_gro_frags(struct napi_struct *napi, struct napi_gro_fraginfo *info)
+void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
+{
+ skb_shinfo(skb)->nr_frags = 0;
+
+ skb->len -= skb->data_len;
+ skb->truesize -= skb->data_len;
+ skb->data_len = 0;
+
+ __skb_pull(skb, skb_headlen(skb));
+ skb_reserve(skb, NET_IP_ALIGN - skb_headroom(skb));
+
+ napi->skb = skb;
+}
+EXPORT_SYMBOL(napi_reuse_skb);
+
+struct sk_buff *napi_fraginfo_skb(struct napi_struct *napi,
+ struct napi_gro_fraginfo *info)
{
struct net_device *dev = napi->dev;
struct sk_buff *skb = napi->skb;
- int err = NET_RX_DROP;
napi->skb = NULL;
skb->len += info->len;
skb->truesize += info->len;
- if (!pskb_may_pull(skb, ETH_HLEN))
- goto reuse;
-
- err = NET_RX_SUCCESS;
+ if (!pskb_may_pull(skb, ETH_HLEN)) {
+ napi_reuse_skb(napi, skb);
+ goto out;
+ }
skb->protocol = eth_type_trans(skb, dev);
skb->ip_summed = info->ip_summed;
skb->csum = info->csum;
+out:
+ return skb;
+}
+EXPORT_SYMBOL(napi_fraginfo_skb);
+
+int napi_gro_frags(struct napi_struct *napi, struct napi_gro_fraginfo *info)
+{
+ struct sk_buff *skb = napi_fraginfo_skb(napi, info);
+ int err = NET_RX_DROP;
+
+ if (!skb)
+ goto out;
+
+ err = NET_RX_SUCCESS;
+
switch (__napi_gro_receive(napi, skb)) {
case -1:
return netif_receive_skb(skb);
goto out;
}
-reuse:
- skb_shinfo(skb)->nr_frags = 0;
-
- skb->len -= skb->data_len;
- skb->truesize -= skb->data_len;
- skb->data_len = 0;
-
- __skb_pull(skb, skb_headlen(skb));
- skb_reserve(skb, NET_IP_ALIGN - skb_headroom(skb));
-
- napi->skb = skb;
+ napi_reuse_skb(napi, skb);
out:
return err;
* There may not be any more sk_buffs coming right now, so push
* any pending DMA copies to hardware
*/
- if (!cpus_empty(net_dma.channel_mask)) {
- int chan_idx;
- for_each_cpu_mask_nr(chan_idx, net_dma.channel_mask) {
- struct dma_chan *chan = net_dma.channels[chan_idx];
- if (chan)
- dma_async_memcpy_issue_pending(chan);
- }
- }
+ dma_issue_pending_all();
#endif
return;
return NOTIFY_OK;
}
-#ifdef CONFIG_NET_DMA
-/**
- * net_dma_rebalance - try to maintain one DMA channel per CPU
- * @net_dma: DMA client and associated data (lock, channels, channel_mask)
- *
- * This is called when the number of channels allocated to the net_dma client
- * changes. The net_dma client tries to have one DMA channel per CPU.
- */
-
-static void net_dma_rebalance(struct net_dma *net_dma)
-{
- unsigned int cpu, i, n, chan_idx;
- struct dma_chan *chan;
-
- if (cpus_empty(net_dma->channel_mask)) {
- for_each_online_cpu(cpu)
- rcu_assign_pointer(per_cpu(softnet_data, cpu).net_dma, NULL);
- return;
- }
-
- i = 0;
- cpu = first_cpu(cpu_online_map);
-
- for_each_cpu_mask_nr(chan_idx, net_dma->channel_mask) {
- chan = net_dma->channels[chan_idx];
-
- n = ((num_online_cpus() / cpus_weight(net_dma->channel_mask))
- + (i < (num_online_cpus() %
- cpus_weight(net_dma->channel_mask)) ? 1 : 0));
-
- while(n) {
- per_cpu(softnet_data, cpu).net_dma = chan;
- cpu = next_cpu(cpu, cpu_online_map);
- n--;
- }
- i++;
- }
-}
-
-/**
- * netdev_dma_event - event callback for the net_dma_client
- * @client: should always be net_dma_client
- * @chan: DMA channel for the event
- * @state: DMA state to be handled
- */
-static enum dma_state_client
-netdev_dma_event(struct dma_client *client, struct dma_chan *chan,
- enum dma_state state)
-{
- int i, found = 0, pos = -1;
- struct net_dma *net_dma =
- container_of(client, struct net_dma, client);
- enum dma_state_client ack = DMA_DUP; /* default: take no action */
-
- spin_lock(&net_dma->lock);
- switch (state) {
- case DMA_RESOURCE_AVAILABLE:
- for (i = 0; i < nr_cpu_ids; i++)
- if (net_dma->channels[i] == chan) {
- found = 1;
- break;
- } else if (net_dma->channels[i] == NULL && pos < 0)
- pos = i;
-
- if (!found && pos >= 0) {
- ack = DMA_ACK;
- net_dma->channels[pos] = chan;
- cpu_set(pos, net_dma->channel_mask);
- net_dma_rebalance(net_dma);
- }
- break;
- case DMA_RESOURCE_REMOVED:
- for (i = 0; i < nr_cpu_ids; i++)
- if (net_dma->channels[i] == chan) {
- found = 1;
- pos = i;
- break;
- }
-
- if (found) {
- ack = DMA_ACK;
- cpu_clear(pos, net_dma->channel_mask);
- net_dma->channels[i] = NULL;
- net_dma_rebalance(net_dma);
- }
- break;
- default:
- break;
- }
- spin_unlock(&net_dma->lock);
-
- return ack;
-}
-
-/**
- * netdev_dma_register - register the networking subsystem as a DMA client
- */
-static int __init netdev_dma_register(void)
-{
- net_dma.channels = kzalloc(nr_cpu_ids * sizeof(struct net_dma),
- GFP_KERNEL);
- if (unlikely(!net_dma.channels)) {
- printk(KERN_NOTICE
- "netdev_dma: no memory for net_dma.channels\n");
- return -ENOMEM;
- }
- spin_lock_init(&net_dma.lock);
- dma_cap_set(DMA_MEMCPY, net_dma.client.cap_mask);
- dma_async_client_register(&net_dma.client);
- dma_async_client_chan_request(&net_dma.client);
- return 0;
-}
-
-#else
-static int __init netdev_dma_register(void) { return -ENODEV; }
-#endif /* CONFIG_NET_DMA */
/**
* netdev_increment_features - increment feature set by one
if (register_pernet_device(&default_device_ops))
goto out;
- netdev_dma_register();
-
open_softirq(NET_TX_SOFTIRQ, net_tx_action);
open_softirq(NET_RX_SOFTIRQ, net_rx_action);
hotcpu_notifier(dev_cpu_callback, 0);
dst_init();
dev_mcast_init();
+ #ifdef CONFIG_NET_DMA
+ dmaengine_get();
+ #endif
rc = 0;
out:
return rc;
projects / linux-2.6-omap-h63xx.git / blobdiff