-
Notifications
You must be signed in to change notification settings - Fork 25
/
af_ktls.c
2177 lines (1767 loc) · 47.8 KB
/
af_ktls.c
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/*
* af_ktls: TLS/DTLS socket
*
* Copyright (C) 2016
*
* Original authors:
* Fridolin Pokorny <fridolin.pokorny@gmail.com>
* Nikos Mavrogiannopoulos <nmav@gnults.org>
* Dave Watson <davejwatson@fb.com>
* Lance Chao <lancerchao@fb.com>
*
* Based on RFC 5288, RFC 6347, RFC 5246, RFC 6655
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License as
* published by the Free Software Foundation; either version 2 of the
* License, or (at your option) any later version.
*/
#include <crypto/aead.h>
#include <crypto/if_alg.h>
#include <linux/init.h>
#include <linux/file.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/net.h>
#include <net/sock.h>
#include <net/tcp.h>
#include <net/strparser.h>
#include <linux/skbuff.h>
#include <linux/log2.h>
#include "af_ktls.h"
#ifndef CHAR_BIT
# define CHAR_BIT 8
#endif
#define MAX(a, b) ((a) > (b) ? (a) : (b))
#define KTLS_RECORD_DATA 0x17
#define KTLS_KEY_SIZE KTLS_AES_GCM_128_KEY_SIZE
#define KTLS_SALT_SIZE KTLS_AES_GCM_128_SALT_SIZE
#define KTLS_TAG_SIZE 16
#define KTLS_IV_SIZE KTLS_AES_GCM_128_IV_SIZE
#define KTLS_NONCE_SIZE 8
#define KTLS_DATA_PAGES (KTLS_MAX_PAYLOAD_SIZE / PAGE_SIZE)
/* +1 for aad, +1 for tag, +1 for chaining */
#define KTLS_SG_DATA_SIZE (KTLS_DATA_PAGES + 3)
#define KTLS_AAD_SPACE_SIZE 21
#define KTLS_AAD_SIZE 13
/* TLS
*/
#define KTLS_TLS_HEADER_SIZE 5
#define KTLS_TLS_PREPEND_SIZE (KTLS_TLS_HEADER_SIZE + KTLS_NONCE_SIZE)
#define KTLS_TLS_OVERHEAD (KTLS_TLS_PREPEND_SIZE + KTLS_TAG_SIZE)
#define KTLS_TLS_1_2_MAJOR 0x03
#define KTLS_TLS_1_2_MINOR 0x03
/* nonce explicit offset in a record */
#define KTLS_TLS_NONCE_OFFSET KTLS_TLS_HEADER_SIZE
#define KTLS_PREPEND_SIZE(T) (IS_TLS(T) ? \
(KTLS_TLS_PREPEND_SIZE) : \
(KTLS_DTLS_PREPEND_SIZE))
#define KTLS_HEADER_SIZE(T) (IS_TLS(T) ? \
(KTLS_TLS_HEADER_SIZE) : \
(KTLS_DTLS_HEADER_SIZE))
#define KTLS_OVERHEAD(T) (IS_TLS(T) ? \
(KTLS_TLS_OVERHEAD) : \
(KTLS_DTLS_OVERHEAD))
/* DTLS
*/
#define KTLS_DTLS_HEADER_SIZE 13
#define KTLS_DTLS_PREPEND_SIZE (KTLS_DTLS_HEADER_SIZE \
+ KTLS_NONCE_SIZE)
#define KTLS_DTLS_OVERHEAD (KTLS_DTLS_PREPEND_SIZE \
+ KTLS_TAG_SIZE)
#define KTLS_DTLS_1_2_MAJOR 0xFE
#define KTLS_DTLS_1_2_MINOR 0xFD
/* we are handling epoch and seq num as one unit */
#define KTLS_DTLS_SEQ_NUM_OFFSET 3
/* nonce explicit offset in a record */
#define KTLS_DTLS_NONCE_OFFSET KTLS_DTLS_HEADER_SIZE
/* Ensure that bind(2) was called
*/
#define KTLS_SETSOCKOPT_READY(T) ((T)->aead_send && (T)->aead_recv)
#define KTLS_GETSOCKOPT_READY(T) KTLS_SETSOCKOPT_READY(T)
/* Ensure that we have needed key material
*/
#define KTLS_SEND_READY(T) ((T)->key_send.keylen && \
(T)->key_send.saltlen && \
(T)->iv_send && \
KTLS_GETSOCKOPT_READY(T))
#define KTLS_RECV_READY(T) ((T)->key_recv.keylen && \
(T)->key_recv.saltlen && \
(T)->iv_recv && \
KTLS_GETSOCKOPT_READY(T))
#define IS_TLS(T) ((T)->sk.sk_type == SOCK_STREAM)
#define IS_DTLS(T) (!IS_TLS(T))
/* Distinguish bound socket type
*/
#define IS_INET46(S) ((S)->sk->sk_family == AF_INET || \
(S)->sk->sk_family == AF_INET6)
#define IS_TCP(S) (IS_INET46(S) && \
(S)->sk->sk_type == SOCK_STREAM)
#define IS_UDP(S) (IS_INET46(S) && \
(S)->sk->sk_type == SOCK_DGRAM)
/* Real size of a record based on data carried
*/
#define KTLS_RECORD_SIZE(T, S) (IS_TLS(T) ? \
(S + KTLS_TLS_OVERHEAD) : \
(S + KTLS_DTLS_OVERHEAD))
/* Nonce explicit offset in a record
*/
#define KTLS_NONCE_OFFSET(T) (IS_TLS(T) ? \
(KTLS_TLS_NONCE_OFFSET) : \
(KTLS_DTLS_NONCE_OFFSET))
/* Async worker */
static struct workqueue_struct *tls_rx_wq;
static struct workqueue_struct *tls_tx_wq;
struct tls_key {
char *key;
size_t keylen;
char salt[KTLS_SALT_SIZE];
size_t saltlen;
};
struct tls_sock {
/* struct sock must be the very first member */
struct sock sk;
/* TCP/UDP socket we are bound to */
struct socket *socket;
int rx_stopped;
/* Context for {set,get}sockopt() */
unsigned char *iv_send;
struct tls_key key_send;
unsigned char *iv_recv;
struct tls_key key_recv;
struct crypto_aead *aead_send;
struct crypto_aead *aead_recv;
/* Sending context */
struct scatterlist sg_tx_data[KTLS_SG_DATA_SIZE];
struct scatterlist sg_tx_data2[ALG_MAX_PAGES + 1];
char aad_send[KTLS_AAD_SPACE_SIZE];
char tag_send[KTLS_TAG_SIZE];
struct page *pages_send;
int send_offset;
int send_len;
int order_npages;
struct scatterlist sgaad_send[2];
struct scatterlist sgtag_send[2];
struct work_struct send_work;
/* Receive */
struct scatterlist sgin[ALG_MAX_PAGES + 1];
char aad_recv[KTLS_AAD_SPACE_SIZE];
char header_recv[MAX(KTLS_TLS_PREPEND_SIZE, KTLS_DTLS_PREPEND_SIZE)];
struct strparser strp;
struct sk_buff_head rx_hold_queue;
struct work_struct recv_work;
void (*saved_sk_data_ready)(struct sock *sk);
void (*saved_sk_write_space)(struct sock *sk);
size_t recv_len;
/* our cipher type and its crypto API representation (e.g. "gcm(aes)")
*/
unsigned int cipher_type;
char *cipher_crypto;
/* TLS/DTLS version for header */
char version[2];
/* DTLS window handling */
struct {
u64 bits;
/* The starting point of the sliding window without epoch */
u64 start;
} dtls_window;
int unsent;
};
struct tls_rx_msg {
/* strp_rx_msg must be first to match strparser */
struct strp_rx_msg rxm;
int decrypted;
};
static inline struct tls_rx_msg *tls_rx_msg(struct sk_buff *skb)
{
return (struct tls_rx_msg *)((void *)skb->cb +
offsetof(struct qdisc_skb_cb, data));
}
static inline struct tls_sock *tls_sk(struct sock *sk)
{
return (struct tls_sock *)sk;
}
static inline bool tls_stream_memory_free(const struct sock *sk)
{
const struct tls_sock *tsk = (const struct tls_sock *)sk;
return tsk->unsent < KTLS_MAX_PAYLOAD_SIZE;
}
static int tls_do_decryption(struct tls_sock *tsk,
struct scatterlist *sgin,
struct scatterlist *sgout,
char *header_recv,
size_t data_len);
static inline void tls_make_aad(struct tls_sock *tsk,
int recv,
char *buf,
size_t size,
char *nonce_explicit);
static int tls_post_process(struct tls_sock *tsk, struct sk_buff *skb);
static void tls_err_abort(struct tls_sock *tsk);
static void increment_seqno(unsigned char *seq, struct tls_sock *tsk)
{
int i;
for (i = 7; i >= 0; i--) {
++seq[i];
if (seq[i] != 0)
break;
}
/* Check for overflow. If overflowed, connection must
* disconnect. Raise an error and notify userspace.
*/
if (unlikely((IS_TLS(tsk) && i == -1) || (IS_DTLS(tsk) && i <= 1)))
tls_err_abort(tsk);
}
/* Must be called with socket callback locked */
static void tls_unattach(struct tls_sock *tsk)
{
write_lock_bh(&tsk->socket->sk->sk_callback_lock);
tsk->rx_stopped = 1;
strp_stop(&tsk->strp);
tsk->socket->sk->sk_data_ready = tsk->saved_sk_data_ready;
tsk->socket->sk->sk_write_space = tsk->saved_sk_write_space;
tsk->socket->sk->sk_user_data = NULL;
write_unlock_bh(&tsk->socket->sk->sk_callback_lock);
}
static void tls_err_abort(struct tls_sock *tsk)
{
struct sock *sk;
sk = (struct sock *)tsk;
xchg(&tsk->rx_stopped, 1);
xchg(&sk->sk_err, -EBADMSG);
sk->sk_error_report(sk);
tsk->saved_sk_data_ready(tsk->socket->sk);
}
static void tls_abort_cb(struct strparser *strp, int err)
{
struct tls_sock *tsk;
tsk = strp->sk->sk_user_data;
if (tsk)
tls_err_abort(tsk);
}
static int decrypt_skb(struct tls_sock *tsk, struct sk_buff *skb)
{
int ret, nsg;
size_t prepend, overhead;
struct strp_rx_msg *rxm;
char header_recv[MAX(KTLS_TLS_PREPEND_SIZE, KTLS_DTLS_PREPEND_SIZE)];
prepend = IS_TLS(tsk) ? KTLS_TLS_PREPEND_SIZE : KTLS_DTLS_PREPEND_SIZE;
overhead = IS_TLS(tsk) ? KTLS_TLS_OVERHEAD : KTLS_DTLS_OVERHEAD;
rxm = strp_rx_msg(skb);
/* Copy header to pass into decryption routine. Cannot use
* tsk->header_recv as that would cause a race between here
* and data_ready
*/
ret = skb_copy_bits(skb, rxm->offset, header_recv, prepend);
if (ret < 0)
goto decryption_fail;
sg_init_table(tsk->sgin, ARRAY_SIZE(tsk->sgin));
sg_set_buf(&tsk->sgin[0], tsk->aad_recv, sizeof(tsk->aad_recv));
nsg = skb_to_sgvec(skb, &tsk->sgin[1], rxm->offset +
prepend,
rxm->full_len - prepend);
/* The length of sg into decryption must not be over
* ALG_MAX_PAGES. The aad takes the first sg, so the payload
* must be less than ALG_MAX_PAGES - 1
*/
if (nsg > ALG_MAX_PAGES - 1) {
ret = -EBADMSG;
goto decryption_fail;
}
tls_make_aad(tsk, 1, tsk->aad_recv,
rxm->full_len - overhead,
tsk->iv_recv);
/* Decrypt in place. After this function call, the decrypted
* data will be in rxm->offset. We must therefore account for
* the fact that the lengths of skbuff_in and skbuff_out are
* different
*/
ret = tls_do_decryption(tsk,
tsk->sgin,
tsk->sgin,
header_recv,
rxm->full_len - overhead);
if (ret < 0)
goto decryption_fail;
ret = tls_post_process(tsk, skb);
if (ret < 0)
goto decryption_fail;
return 0;
decryption_fail:
return ret;
}
/* Returns the length of the unencrypted message, plus overhead Note
* that this function also populates tsk->header which is later used
* for decryption. In TLS we automatically bail if we see a non-TLS
* message. In DTLS we should determine if we got a corrupted message
* vs a control msg Right now if the TLS magic bit got corrupted it
* would incorrectly misinterpret it as a non-TLS message Returns 0 if
* more data is necessary to determine length Returns <0 if error
* occurred
*/
static inline ssize_t tls_read_size(struct tls_sock *tsk, struct sk_buff *skb)
{
int ret;
size_t data_len = 0;
size_t datagram_len;
size_t prepend;
char first_byte;
char *header;
struct strp_rx_msg *rxm;
prepend = IS_TLS(tsk) ? KTLS_TLS_PREPEND_SIZE : KTLS_DTLS_PREPEND_SIZE;
header = tsk->header_recv;
rxm = strp_rx_msg(skb);
ret = skb_copy_bits(skb, rxm->offset, &first_byte, 1);
if (ret < 0)
goto read_failure;
/* Check the first byte to see if its a TLS record */
if (first_byte != KTLS_RECORD_DATA) {
ret = -EBADMSG;
goto read_failure;
}
/* We have a TLS record. Check that msglen is long enough to
* read the length of record. We must not check this before
* checking the first byte, since that will cause unencrypted
* messages shorter than KTLS_TLS_PREPEND_SIZE to not be read
*/
if (rxm->offset + prepend > skb->len) {
ret = 0;
goto read_failure;
}
/* Copy header to read size. An optimization could be to
* zero-copy, but you'd have to be able to walk
* frag_lists. This function call takes care of that.
* Overhead is relatively small (13 bytes for TLS, 21 for
* DTLS)
*/
ret = skb_copy_bits(skb, rxm->offset, header, prepend);
if (ret < 0)
goto read_failure;
if (IS_TLS(tsk)) {
data_len = ((header[4] & 0xFF) | (header[3] << 8));
data_len = data_len - KTLS_TAG_SIZE - KTLS_IV_SIZE;
datagram_len = data_len + KTLS_TLS_OVERHEAD;
} else {
data_len = ((header[12] & 0xFF) | (header[11] << 8));
data_len = data_len - KTLS_TAG_SIZE - KTLS_IV_SIZE;
datagram_len = data_len + KTLS_DTLS_OVERHEAD;
}
if (data_len > KTLS_MAX_PAYLOAD_SIZE) {
ret = -E2BIG;
goto read_failure;
}
return datagram_len;
read_failure:
/* TLS couldn't handle this message. Pass it directly to userspace */
if (ret == -EBADMSG)
tls_err_abort(tsk);
return ret;
}
static int tls_parse_cb(struct strparser *strp, struct sk_buff *skb)
{
struct tls_sock *tsk;
tsk = strp->sk->sk_user_data;
if (tsk)
return tls_read_size(tsk, skb);
else
return -1;
}
static void tls_queue(struct strparser *strp, struct sk_buff *skb)
{
struct tls_sock *tsk;
int ret;
struct strp_rx_msg *rxm;
rxm = strp_rx_msg(skb);
tsk = strp->sk->sk_user_data;
if (!tsk || tsk->rx_stopped) {
kfree_skb(skb);
return;
}
tls_rx_msg(skb)->decrypted = 0;
bh_lock_sock(&tsk->sk);
tsk->recv_len += rxm->full_len - KTLS_OVERHEAD(tsk);
ret = sock_queue_rcv_skb((struct sock *)tsk, skb);
if (ret < 0) {
/* skb receive queue is full. Apply backpressure on
* TCP socket
*/
skb_queue_tail(&tsk->rx_hold_queue, skb);
strp->rx_paused = 1;
tsk->sk.sk_data_ready(&tsk->sk);
}
bh_unlock_sock(&tsk->sk);
}
/* Called with lower socket held */
static void tls_data_ready(struct sock *sk)
{
struct tls_sock *tsk;
read_lock_bh(&sk->sk_callback_lock);
tsk = (struct tls_sock *)sk->sk_user_data;
if (unlikely(!tsk || tsk->rx_stopped))
goto out;
if (IS_TLS(tsk))
strp_data_ready(&tsk->strp);
else
queue_work(tls_rx_wq, &tsk->recv_work);
out:
read_unlock_bh(&sk->sk_callback_lock);
}
/* Called with lower socket held */
static void tls_write_space(struct sock *sk)
{
struct tls_sock *tsk;
read_lock_bh(&sk->sk_callback_lock);
tsk = (struct tls_sock *)sk->sk_user_data;
if (unlikely(!tsk || tsk->rx_stopped))
goto out;
queue_work(tls_tx_wq, &tsk->send_work);
out:
read_unlock_bh(&sk->sk_callback_lock);
}
#include "dtls-window.c"
/* Loop through the SKBs. Decrypt each one and, if valid, add it to recv queue
*/
static int dtls_udp_read_sock(struct tls_sock *tsk)
{
struct sk_buff *p, *next, *skb;
int ret = 0;
skb_queue_walk_safe(&tsk->socket->sk->sk_receive_queue, p, next) {
ssize_t len;
struct strp_rx_msg *rxm;
rxm = strp_rx_msg(p);
memset(rxm, 0, sizeof(*rxm));
/* For UDP, set the offset such that the headers are
* ignored. Full_len is length of skb minus the
* headers
*/
rxm->full_len = p->len - sizeof(struct udphdr);
rxm->offset = sizeof(struct udphdr);
len = tls_read_size(tsk, p);
if (!len)
goto record_pop;
if (len < 0) {
if (len == -EBADMSG) {
/* Data does not appear to be a TLS
* record Make userspace handle it
*/
ret = -EBADMSG;
break;
}
/* Failed for some other reason. Drop the
* packet
*/
goto record_pop;
}
if (dtls_window(tsk, tsk->header_recv +
KTLS_DTLS_SEQ_NUM_OFFSET) < 0)
goto record_pop;
skb = skb_clone(p, GFP_ATOMIC);
if (!skb) {
ret = -ENOMEM;
break;
}
sock_queue_rcv_skb((struct sock *)tsk, skb);
record_pop:
skb_unlink(p, &tsk->socket->sk->sk_receive_queue);
kfree_skb(p);
}
return ret;
}
static void do_dtls_data_ready(struct tls_sock *tsk)
{
int ret;
ret = dtls_udp_read_sock(tsk);
if (ret == -ENOMEM) /* No memory. Do it later */
queue_work(tls_rx_wq, &tsk->recv_work);
/* TLS couldn't handle this message. Pass it directly to
* userspace
*/
else if (ret == -EBADMSG)
tls_err_abort(tsk);
}
static void do_dtls_sock_rx_work(struct tls_sock *tsk)
{
struct sock *sk = tsk->socket->sk;
lock_sock(sk);
read_lock_bh(&sk->sk_callback_lock);
if (unlikely(!tsk || sk->sk_user_data != tsk))
goto out;
if (unlikely(tsk->rx_stopped))
goto out;
if (!KTLS_RECV_READY(tsk))
goto out;
do_dtls_data_ready(tsk);
out:
read_unlock_bh(&sk->sk_callback_lock);
release_sock(sk);
}
static void check_rcv(struct tls_sock *tsk)
{
if (IS_TLS(tsk))
strp_check_rcv(&tsk->strp);
else
do_dtls_sock_rx_work(tsk);
}
static void tls_rx_work(struct work_struct *w)
{
do_dtls_sock_rx_work(container_of(w, struct tls_sock, recv_work));
}
static void tls_kernel_sendpage(struct tls_sock *tsk);
static void tls_tx_work(struct work_struct *w)
{
struct tls_sock *tsk = container_of(w, struct tls_sock, send_work);
struct sock *sk = &tsk->sk;
lock_sock(sk);
tls_kernel_sendpage(tsk);
release_sock(sk);
}
static int tls_set_iv(struct socket *sock,
int recv,
char __user *src,
size_t src_len)
{
int ret;
unsigned char **iv;
struct sock *sk;
struct tls_sock *tsk;
sk = sock->sk;
tsk = tls_sk(sk);
if (!src)
return -EBADMSG;
if (src_len != KTLS_IV_SIZE)
return -EBADMSG;
iv = recv ? &tsk->iv_recv : &tsk->iv_send;
if (!*iv) {
*iv = kmalloc(src_len, GFP_KERNEL);
if (!*iv)
return -ENOMEM;
}
ret = copy_from_user(*iv, src, src_len);
return ret ?: src_len;
}
static int tls_init_aead(struct tls_sock *tsk, int recv)
{
int ret;
struct crypto_aead *aead;
struct tls_key *k;
char keyval[KTLS_KEY_SIZE + KTLS_SALT_SIZE];
size_t keyval_len;
k = recv ? &tsk->key_recv : &tsk->key_send;
aead = recv ? tsk->aead_recv : tsk->aead_send;
/* We need salt and key in order to construct 20B key
* according to RFC5288, otherwise we will handle this once
* both will be provided
*/
if (k->keylen == 0 || k->saltlen == 0)
return 0;
keyval_len = k->keylen + k->saltlen;
memcpy(keyval, k->key, k->keylen);
memcpy(keyval + k->keylen, k->salt, k->saltlen);
ret = crypto_aead_setkey(aead, keyval, keyval_len);
if (ret)
goto init_aead_end;
ret = crypto_aead_setauthsize(aead, KTLS_TAG_SIZE);
init_aead_end:
return ret ?: 0;
}
static int tls_set_key(struct socket *sock,
int recv,
char __user *src,
size_t src_len)
{
int ret;
struct tls_sock *tsk;
struct tls_key *k;
tsk = tls_sk(sock->sk);
if (src_len == 0 || !src)
return -EBADMSG;
if (src_len != KTLS_KEY_SIZE)
return -EBADMSG;
k = recv ? &tsk->key_recv : &tsk->key_send;
if (src_len > k->keylen) {
kfree(k->key);
k->key = kmalloc(src_len, GFP_KERNEL);
if (!k->key)
return -ENOMEM;
}
ret = copy_from_user(k->key, src, src_len);
if (ret)
goto set_key_end;
k->keylen = src_len;
ret = tls_init_aead(tsk, recv);
set_key_end:
return ret < 0 ? ret : src_len;
}
static int tls_set_salt(struct socket *sock,
int recv,
char __user *src,
size_t src_len)
{
int ret;
struct tls_sock *tsk;
struct tls_key *k;
tsk = tls_sk(sock->sk);
k = recv ? &tsk->key_recv : &tsk->key_send;
if (src_len != KTLS_SALT_SIZE)
return -EBADMSG;
ret = copy_from_user(k->salt, src, src_len);
if (ret)
goto set_salt_end;
k->saltlen = src_len;
ret = tls_init_aead(tsk, recv);
set_salt_end:
return ret < 0 ? ret : src_len;
}
static void tls_do_unattach(struct socket *sock)
{
struct tls_sock *tsk;
struct sock *sk;
tsk = tls_sk(sock->sk);
sk = tsk->socket->sk;
tls_unattach(tsk);
}
static int tls_setsockopt(struct socket *sock,
int level, int optname,
char __user *optval,
unsigned int optlen)
{
int ret;
struct tls_sock *tsk;
tsk = tls_sk(sock->sk);
if (level != AF_KTLS)
return -ENOPROTOOPT;
lock_sock(sock->sk);
ret = -EBADMSG;
if (!KTLS_SETSOCKOPT_READY(tsk))
goto setsockopt_end;
switch (optname) {
case KTLS_SET_IV_RECV:
ret = tls_set_iv(sock, 1, optval, optlen);
break;
case KTLS_SET_KEY_RECV:
ret = tls_set_key(sock, 1, optval, optlen);
break;
case KTLS_SET_SALT_RECV:
ret = tls_set_salt(sock, 1, optval, optlen);
break;
case KTLS_SET_IV_SEND:
ret = tls_set_iv(sock, 0, optval, optlen);
break;
case KTLS_SET_KEY_SEND:
ret = tls_set_key(sock, 0, optval, optlen);
break;
case KTLS_SET_SALT_SEND:
ret = tls_set_salt(sock, 0, optval, optlen);
break;
case KTLS_UNATTACH:
tls_do_unattach(sock);
ret = 0;
break;
default:
break;
}
setsockopt_end:
release_sock(sock->sk);
return ret < 0 ? ret : 0;
}
static int tls_get_iv(const struct tls_sock *tsk,
int recv,
char __user *dst,
size_t dst_len)
{
int ret;
char *iv;
if (dst_len < KTLS_IV_SIZE)
return -ENOMEM;
iv = recv ? tsk->iv_recv : tsk->iv_send;
if (!iv)
return -EBADMSG;
ret = copy_to_user(dst, iv, KTLS_IV_SIZE);
if (ret)
return ret;
return KTLS_IV_SIZE;
}
static int tls_get_key(const struct tls_sock *tsk,
int recv,
char __user *dst,
size_t dst_len)
{
int ret;
const struct tls_key *k;
k = recv ? &tsk->key_recv : &tsk->key_send;
if (k->keylen == 0)
return -EBADMSG;
if (dst_len < k->keylen)
return -ENOMEM;
ret = copy_to_user(dst, k->key, k->keylen);
return ret ?: k->keylen;
}
static int tls_get_salt(const struct tls_sock *tsk,
int recv,
char __user *dst,
size_t dst_len)
{
int ret;
const struct tls_key *k;
k = recv ? &tsk->key_recv : &tsk->key_send;
if (k->saltlen == 0)
return -EBADMSG;
if (dst_len < k->saltlen)
return -ENOMEM;
ret = copy_to_user(dst, k->salt, k->saltlen);
return ret ?: k->saltlen;
}
static int tls_getsockopt(struct socket *sock,
int level,
int optname,
char __user *optval,
int __user *optlen)
{
int ret;
int len;
const struct tls_sock *tsk;
tsk = tls_sk(sock->sk);
if (level != AF_KTLS)
return -ENOPROTOOPT;
if (!optlen || !optval)
return -EBADMSG;
if (get_user(len, optlen))
return -EFAULT;
lock_sock(sock->sk);
ret = -EBADMSG;
if (!KTLS_GETSOCKOPT_READY(tsk))
goto end;
switch (optname) {
case KTLS_GET_IV_RECV:
ret = tls_get_iv(tsk, 1, optval, len);
break;
case KTLS_GET_KEY_RECV:
ret = tls_get_key(tsk, 1, optval, len);
break;
case KTLS_GET_SALT_RECV:
ret = tls_get_salt(tsk, 1, optval, len);
break;
case KTLS_GET_IV_SEND:
ret = tls_get_iv(tsk, 0, optval, len);
break;
case KTLS_GET_KEY_SEND:
ret = tls_get_key(tsk, 0, optval, len);
break;
case KTLS_GET_SALT_SEND:
ret = tls_get_salt(tsk, 0, optval, len);
break;
default:
ret = -EBADMSG;
break;
}
if (ret < 0)
goto end;
ret = copy_to_user(optlen, &ret, sizeof(*optlen));
end:
release_sock(sock->sk);
return ret;
}
static inline void tls_make_prepend(struct tls_sock *tsk,
char *buf,
size_t plaintext_len)
{
size_t pkt_len;
pkt_len = plaintext_len + KTLS_IV_SIZE + KTLS_TAG_SIZE;
/* we cover nonce explicit here as well, so buf should be of
* size KTLS_DTLS_HEADER_SIZE + KTLS_DTLS_NONCE_EXPLICIT_SIZE
*/
buf[0] = KTLS_RECORD_DATA;
buf[1] = tsk->version[0];
buf[2] = tsk->version[1];
/* we can use IV for nonce explicit according to spec */
if (IS_TLS(tsk)) {
buf[3] = pkt_len >> 8;
buf[4] = pkt_len & 0xFF;
memcpy(buf + KTLS_TLS_NONCE_OFFSET, tsk->iv_send, KTLS_IV_SIZE);
} else {
memcpy(buf + 3, tsk->iv_send, KTLS_IV_SIZE);
buf[11] = pkt_len >> 8;
buf[12] = pkt_len & 0xFF;
memcpy(buf + KTLS_DTLS_NONCE_OFFSET,
tsk->iv_send,
KTLS_IV_SIZE);
}
}
static inline void tls_make_aad(struct tls_sock *tsk,
int recv,
char *buf,
size_t size,
char *nonce_explicit)
{
memcpy(buf, nonce_explicit, KTLS_NONCE_SIZE);
buf[8] = KTLS_RECORD_DATA;
buf[9] = tsk->version[0];
buf[10] = tsk->version[1];
buf[11] = size >> 8;
buf[12] = size & 0xFF;
}