diff --git a/homa_incoming.c b/homa_incoming.c index cbf06730..008f4e68 100644 --- a/homa_incoming.c +++ b/homa_incoming.c @@ -715,6 +715,7 @@ void homa_data_pkt(struct sk_buff *skb, struct homa_rpc *rpc) { enum skb_drop_reason drop_reason = SKB_DROP_REASON_NOT_SPECIFIED; struct homa_data_hdr *h = (struct homa_data_hdr *)skb->data; + bool batch_end = !skb->next; /* Before skb is queued in msgin. */ IF_NO_STRIP(struct homa *homa = rpc->hsk->homa); @@ -792,20 +793,19 @@ void homa_data_pkt(struct sk_buff *skb, struct homa_rpc *rpc) drop_reason = homa_add_packet(rpc, skb); - if (skb_queue_len(&rpc->msgin.packets) != 0 && + /* Hand off only on the last packet of the batch (see + * homa_dispatch_pkts): a thread woken earlier would ask for the RPC + * lock while the rest of the batch still needs it. + */ + if (batch_end && skb_queue_len(&rpc->msgin.packets) != 0 && !test_bit(RPC_PKTS_READY, &rpc->flags)) { set_bit(RPC_PKTS_READY, &rpc->flags); homa_rpc_handoff(rpc); } handle_ack: - /* It's important to handle acks *after* calling homa_add_packet. - * Otherwise we can end up with extraneous handoffs: we have to release - * the RPC lock to handle acks, but that could allow some other thread - * to receive a previous handoff made by homa_rpc_alloc_server, even - * though the packet hasn't been added to the RPC. If this happens - * the call to homa_rpc_handoff above will have to make an additional - * handoff. + /* Handling acks releases the RPC lock, so the packet is queued + * first: a thread that copies from the RPC meanwhile will find it. */ if (h->ack.client_id) { const struct in6_addr saddr = skb_canonical_ipv6_saddr(skb); diff --git a/homa_rpc.c b/homa_rpc.c index bdbe8752..b0636d82 100644 --- a/homa_rpc.c +++ b/homa_rpc.c @@ -97,9 +97,7 @@ struct homa_rpc *homa_rpc_alloc_client(struct homa_sock *hsk, /** * homa_rpc_alloc_server() - Allocate and initialize a server RPC (one that is - * used to manage an incoming request). If appropriate, the RPC will also - * be handed off (we do it here, while we have the socket locked, to avoid - * acquiring the socket lock a second time later for the handoff). + * used to manage an incoming request). * @hsk: Socket that owns this RPC. * @source: IP address (network byte order) of the RPC's client. * @h: Header for the first data packet received for this RPC; used @@ -210,10 +208,6 @@ struct homa_rpc *homa_rpc_alloc_server(struct homa_sock *hsk, hlist_add_head(&srpc->hash_links, &bucket->rpcs); list_add_tail_rcu(&srpc->active_links, &hsk->active_rpcs); homa_sock_unlock(hsk); - if (h->type == DATA && srpc->msgin.num_bpages > 0) { - set_bit(RPC_PKTS_READY, &srpc->flags); - homa_rpc_handoff(srpc); - } INC_METRIC(requests_received, 1); return srpc; diff --git a/test/unit_homa_incoming.c b/test/unit_homa_incoming.c index 56230230..131bf3dd 100644 --- a/test/unit_homa_incoming.c +++ b/test/unit_homa_incoming.c @@ -49,6 +49,14 @@ static void wait_hook4(char *id) homa_rpc_handoff(hook_rpc); } +static void count_packets_data_ready(struct sock *sk) +{ + struct homa_rpc *rpc = list_first_entry(&homa_sk(sk)->active_rpcs, + struct homa_rpc, active_links); + + hook_count = skb_queue_len(&rpc->msgin.packets); +} + static void handoff_hook(char *id) { if (strcmp(id, "spin_lock") != 0) @@ -1307,6 +1315,22 @@ TEST_F(homa_incoming, homa_dispatch_pkts__cant_create_server_rpc) EXPECT_EQ(1, homa_metrics_per_cpu()->server_cant_create_rpcs); #endif /* See strip.py */ } +TEST_F(homa_incoming, homa_dispatch_pkts__handoff_after_batch) +{ + struct sk_buff *skb; + + self->data.msg_length = htonl(3000); + skb = mock_skb_alloc(self->client_ip, self->server_ip, + &self->data.common, 1400, 0); + self->data.seg.offset = htonl(1400); + skb->next = mock_skb_alloc(self->client_ip, self->server_ip, + &self->data.common, 1400, 1400); + self->hsk2.sock.sk_data_ready = count_packets_data_ready; + hook_count = 0; + homa_dispatch_pkts(skb); + EXPECT_EQ(1, unit_list_length(&self->hsk2.ready_rpcs)); + EXPECT_EQ(2, hook_count); +} TEST_F(homa_incoming, homa_dispatch_pkts__existing_server_rpc) { struct homa_rpc *srpc = unit_server_rpc(&self->hsk2, UNIT_RCVD_ONE_PKT, diff --git a/test/unit_homa_rpc.c b/test/unit_homa_rpc.c index d201be6e..a5bb136a 100644 --- a/test/unit_homa_rpc.c +++ b/test/unit_homa_rpc.c @@ -261,10 +261,11 @@ TEST_F(homa_rpc, homa_rpc_alloc_server__socket_shutdown) EXPECT_EQ(0, unit_list_length(&self->hsk.active_rpcs)); self->hsk.shutdown = 0; } -TEST_F(homa_rpc, homa_rpc_alloc_server__handoff_rpc) +TEST_F(homa_rpc, homa_rpc_alloc_server__no_handoff) { struct homa_rpc *srpc; + /* homa_data_pkt hands the message over, not homa_rpc_alloc_server. */ self->data.msg_length = N(1400); srpc = homa_rpc_alloc_server(&self->hsk, self->client_ip, &self->data.common); @@ -272,43 +273,9 @@ TEST_F(homa_rpc, homa_rpc_alloc_server__handoff_rpc) homa_rpc_unlock(srpc); EXPECT_EQ(RPC_INCOMING, srpc->state); EXPECT_EQ(1, unit_list_length(&self->hsk.active_rpcs)); - EXPECT_EQ(1, unit_list_length(&self->hsk.ready_rpcs)); - homa_rpc_end(srpc); -} -TEST_F(homa_rpc, homa_rpc_alloc_server__dont_handoff_no_buffers) -{ - struct homa_rpc *srpc; - - mock_check_bpool_leaks = false; - self->data.msg_length = N(1400); - self->hsk.buffer_pool->free_bpages = 0; - srpc = homa_rpc_alloc_server(&self->hsk, self->client_ip, - &self->data.common); - ASSERT_FALSE(IS_ERR(srpc)); - homa_rpc_unlock(srpc); - EXPECT_EQ(0, unit_list_length(&self->hsk.ready_rpcs)); - homa_rpc_end(srpc); -} -#ifndef __STRIP__ /* See strip.py */ -TEST_F(homa_rpc, homa_rpc_alloc_server__dont_handoff_rpc_start_msg) -{ - struct homa_start_msg_hdr h; - struct homa_rpc *srpc; - - h.common.sport = n(self->client_port); - h.common.dport = n(self->server_port); - h.common.type = START_MSG; - h.common.sender_id = cpu_to_be64(self->client_id); - h.msg_length = N(3000); - srpc = homa_rpc_alloc_server(&self->hsk, self->client_ip, &h.common); - ASSERT_FALSE(IS_ERR(srpc)); - homa_rpc_unlock(srpc); - EXPECT_EQ(RPC_INCOMING, srpc->state); - EXPECT_EQ(1, unit_list_length(&self->hsk.active_rpcs)); EXPECT_EQ(0, unit_list_length(&self->hsk.ready_rpcs)); homa_rpc_end(srpc); } -#endif /* See strip.py */ TEST_F(homa_rpc, homa_rpc_alloc_server__unlink_from_waiting_for_bufs_after_error) { struct homa_rpc *srpc;