Annotation of src/sys/kern/subr_msan.c, Revision 1.2
1.2 ! maxv 1: /* $NetBSD: subr_msan.c,v 1.1 2019/11/14 16:23:52 maxv Exp $ */
1.1 maxv 2:
3: /*
4: * Copyright (c) 2019 The NetBSD Foundation, Inc.
5: * All rights reserved.
6: *
7: * This code is derived from software contributed to The NetBSD Foundation
8: * by Maxime Villard.
9: *
10: * Redistribution and use in source and binary forms, with or without
11: * modification, are permitted provided that the following conditions
12: * are met:
13: * 1. Redistributions of source code must retain the above copyright
14: * notice, this list of conditions and the following disclaimer.
15: * 2. Redistributions in binary form must reproduce the above copyright
16: * notice, this list of conditions and the following disclaimer in the
17: * documentation and/or other materials provided with the distribution.
18: *
19: * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20: * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21: * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22: * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23: * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24: * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25: * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26: * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27: * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28: * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29: * POSSIBILITY OF SUCH DAMAGE.
30: */
31:
32: #define KMSAN_NO_INST
33:
34: #include <sys/cdefs.h>
1.2 ! maxv 35: __KERNEL_RCSID(0, "$NetBSD: subr_msan.c,v 1.1 2019/11/14 16:23:52 maxv Exp $");
1.1 maxv 36:
37: #include <sys/param.h>
38: #include <sys/device.h>
39: #include <sys/kernel.h>
40: #include <sys/param.h>
41: #include <sys/conf.h>
42: #include <sys/systm.h>
43: #include <sys/types.h>
44: #include <sys/kprintf.h>
45: #include <sys/kmem.h>
46: #include <sys/mbuf.h>
47: #include <sys/buf.h>
48: #include <sys/cpu.h>
49: #include <sys/msan.h>
50:
51: #include <uvm/uvm.h>
52:
53: static void kmsan_printf(const char *, ...);
54:
55: #ifdef KMSAN_PANIC
56: #define REPORT panic
57: #else
58: #define REPORT kmsan_printf
59: #endif
60:
61: /* -------------------------------------------------------------------------- */
62:
63: /*
64: * Part of the compiler ABI.
65: */
66:
67: typedef uint32_t msan_orig_t;
68:
69: typedef struct {
70: uint8_t *shad;
71: msan_orig_t *orig;
72: } msan_meta_t;
73:
74: #define MSAN_PARAM_SIZE 800
75: #define MSAN_RETVAL_SIZE 800
76: typedef struct {
77: uint8_t param[MSAN_PARAM_SIZE];
78: uint8_t retval[MSAN_RETVAL_SIZE];
79: uint8_t _va_arg[MSAN_PARAM_SIZE];
80: uint8_t va_arg_origin[MSAN_PARAM_SIZE];
81: uint64_t va_arg_overflow_size;
82: msan_orig_t param_origin[MSAN_PARAM_SIZE];
83: msan_orig_t retval_origin;
84: msan_orig_t origin;
85: } msan_tls_t;
86:
87: /* -------------------------------------------------------------------------- */
88:
89: /* The MD code. */
90: #include <machine/msan.h>
91:
92: /* -------------------------------------------------------------------------- */
93:
94: #define __RET_ADDR (uintptr_t)__builtin_return_address(0)
95: #define MSAN_NCONTEXT 16
96:
97: typedef struct {
98: size_t ctx;
99: msan_tls_t tls[MSAN_NCONTEXT];
100: } msan_lwp_t;
101:
102: static msan_tls_t dummy_tls;
103:
104: static uint8_t msan_dummy_shad[PAGE_SIZE] __aligned(PAGE_SIZE);
105: static uint8_t msan_dummy_orig[PAGE_SIZE] __aligned(PAGE_SIZE);
106: static msan_lwp_t msan_lwp0;
107: static bool kmsan_enabled __read_mostly;
108:
109: /* -------------------------------------------------------------------------- */
110:
111: static bool kmsan_reporting = false;
112:
113: static inline void
114: kmsan_printf(const char *fmt, ...)
115: {
116: va_list ap;
117:
118: va_start(ap, fmt);
119: kprintf(fmt, TOCONS, NULL, NULL, ap);
120: va_end(ap);
121: }
122:
123: static inline const char *
124: kmsan_orig_name(int type)
125: {
126: switch (type) {
127: case KMSAN_TYPE_STACK:
128: return "Stack";
129: case KMSAN_TYPE_KMEM:
130: return "Kmem";
131: case KMSAN_TYPE_MALLOC:
132: return "Malloc";
133: case KMSAN_TYPE_POOL:
134: return "Pool";
135: case KMSAN_TYPE_UVM:
136: return "Uvm";
137: default:
138: return "Unknown";
139: }
140: }
141:
142: /*
143: * The format of the string is: "----var@function". Parse it to display a nice
144: * warning.
145: */
146: static void
147: kmsan_report_hook(const void *addr, size_t size, size_t off, const char *hook)
148: {
149: const char *mod, *sym;
150: extern int db_active;
151: msan_orig_t *orig;
152: const char *typename;
153: char *var, *fn;
154: uintptr_t ptr;
155: char buf[128];
156: int type;
157:
158: if (__predict_false(panicstr != NULL || db_active || kmsan_reporting))
159: return;
160:
161: kmsan_reporting = true;
162: __insn_barrier();
163:
164: orig = (msan_orig_t *)kmsan_md_addr_to_orig(addr);
165: orig = (msan_orig_t *)((uintptr_t)orig & ~0x3);
166:
167: if (*orig == 0) {
168: REPORT("MSan: Uninitialized Memory In %s() At Offset "
169: "%zu\n", hook, off);
170: goto out;
171: }
172:
173: kmsan_md_orig_decode(*orig, &type, &ptr);
174: typename = kmsan_orig_name(type);
175:
176: if (kmsan_md_is_pc(ptr)) {
177: if (ksyms_getname(&mod, &sym, (vaddr_t)ptr, KSYMS_PROC)) {
178: REPORT("MSan: Uninitialized %s Memory In %s() "
179: "At Offset %zu, IP %p\n", typename, hook, off,
180: (void *)ptr);
181: } else {
182: REPORT("MSan: Uninitialized %s Memory In %s() "
183: "At Offset %zu, From %s()\n", typename, hook, off,
184: sym);
185: }
186: } else {
187: var = (char *)ptr + 4;
188: strlcpy(buf, var, sizeof(buf));
189: var = buf;
190: fn = strchr(buf, '@');
191: *fn++ = '\0';
192: REPORT("MSan: Uninitialized %s Memory In %s() At Offset "
193: "%zu, Variable '%s' From %s()\n", typename, hook, off,
194: var, fn);
195: }
196:
197: out:
198: kmsan_md_unwind();
199: __insn_barrier();
200: kmsan_reporting = false;
201: }
202:
203: static void
204: kmsan_report_inline(msan_orig_t orig, unsigned long pc)
205: {
206: const char *mod, *sym;
207: extern int db_active;
208: const char *typename;
209: char *var, *fn;
210: uintptr_t ptr;
211: char buf[128];
212: int type;
213:
214: if (__predict_false(panicstr != NULL || db_active || kmsan_reporting))
215: return;
216:
217: kmsan_reporting = true;
218: __insn_barrier();
219:
220: if (orig == 0) {
221: REPORT("MSan: Uninitialized Variable In %p\n",
222: (void *)pc);
223: goto out;
224: }
225:
226: kmsan_md_orig_decode(orig, &type, &ptr);
227: typename = kmsan_orig_name(type);
228:
229: if (kmsan_md_is_pc(ptr)) {
230: if (ksyms_getname(&mod, &sym, (vaddr_t)ptr, KSYMS_PROC)) {
231: REPORT("MSan: Uninitialized %s Memory, "
232: "Origin %x\n", typename, orig);
233: } else {
234: REPORT("MSan: Uninitialized %s Memory "
235: "From %s()\n", typename, sym);
236: }
237: } else {
238: var = (char *)ptr + 4;
239: strlcpy(buf, var, sizeof(buf));
240: var = buf;
241: fn = strchr(buf, '@');
242: *fn++ = '\0';
243: REPORT("MSan: Uninitialized Variable '%s' From %s()\n",
244: var, fn);
245: }
246:
247: out:
248: kmsan_md_unwind();
249: __insn_barrier();
250: kmsan_reporting = false;
251: }
252:
253: /* -------------------------------------------------------------------------- */
254:
255: static inline msan_meta_t
256: kmsan_meta_get(void *addr, size_t size)
257: {
258: msan_meta_t ret;
259:
260: if (__predict_false(!kmsan_enabled)) {
261: ret.shad = msan_dummy_shad;
262: ret.orig = (msan_orig_t *)msan_dummy_orig;
263: } else if (__predict_false(kmsan_md_unsupported((vaddr_t)addr))) {
264: ret.shad = msan_dummy_shad;
265: ret.orig = (msan_orig_t *)msan_dummy_orig;
266: } else {
267: ret.shad = (void *)kmsan_md_addr_to_shad(addr);
268: ret.orig = (msan_orig_t *)kmsan_md_addr_to_orig(addr);
269: ret.orig = (msan_orig_t *)((uintptr_t)ret.orig & ~0x3);
270: }
271:
272: return ret;
273: }
274:
275: static inline void
276: kmsan_origin_fill(void *addr, msan_orig_t o, size_t size)
277: {
278: msan_orig_t *orig;
279: size_t i;
280:
281: if (__predict_false(!kmsan_enabled))
282: return;
283: if (__predict_false(kmsan_md_unsupported((vaddr_t)addr)))
284: return;
285:
286: orig = (msan_orig_t *)kmsan_md_addr_to_orig(addr);
287: size += ((uintptr_t)orig & 0x3);
288: orig = (msan_orig_t *)((uintptr_t)orig & ~0x3);
289:
290: for (i = 0; i < size; i += 4) {
291: orig[i / 4] = o;
292: }
293: }
294:
295: static inline void
296: kmsan_shadow_fill(void *addr, uint8_t c, size_t size)
297: {
298: uint8_t *shad;
299:
300: if (__predict_false(!kmsan_enabled))
301: return;
302: if (__predict_false(kmsan_md_unsupported((vaddr_t)addr)))
303: return;
304:
305: shad = kmsan_md_addr_to_shad(addr);
306: __builtin_memset(shad, c, size);
307: }
308:
309: static inline void
310: kmsan_meta_copy(void *dst, const void *src, size_t size)
311: {
312: uint8_t *orig_src, *orig_dst;
313: uint8_t *shad_src, *shad_dst;
314: msan_orig_t *_src, *_dst;
315: size_t i;
316:
317: if (__predict_false(!kmsan_enabled))
318: return;
319: if (__predict_false(kmsan_md_unsupported((vaddr_t)dst)))
320: return;
321: if (__predict_false(kmsan_md_unsupported((vaddr_t)src))) {
322: kmsan_shadow_fill(dst, KMSAN_STATE_INITED, size);
323: return;
324: }
325:
326: shad_src = kmsan_md_addr_to_shad(src);
327: shad_dst = kmsan_md_addr_to_shad(dst);
328: __builtin_memmove(shad_dst, shad_src, size);
329:
330: orig_src = kmsan_md_addr_to_orig(src);
331: orig_dst = kmsan_md_addr_to_orig(dst);
332: for (i = 0; i < size; i++) {
333: _src = (msan_orig_t *)((uintptr_t)orig_src & ~0x3);
334: _dst = (msan_orig_t *)((uintptr_t)orig_dst & ~0x3);
335: *_dst = *_src;
336: orig_src++;
337: orig_dst++;
338: }
339: }
340:
341: static inline void
342: kmsan_shadow_check(const void *addr, size_t size, const char *hook)
343: {
344: uint8_t *shad;
345: size_t i;
346:
347: if (__predict_false(!kmsan_enabled))
348: return;
349: if (__predict_false(kmsan_md_unsupported((vaddr_t)addr)))
350: return;
351:
352: shad = kmsan_md_addr_to_shad(addr);
353: for (i = 0; i < size; i++) {
354: if (__predict_true(shad[i] == 0))
355: continue;
356: kmsan_report_hook((const char *)addr + i, size, i, hook);
357: break;
358: }
359: }
360:
361: void kmsan_init_arg(size_t);
362: void kmsan_init_ret(size_t);
363:
364: void
365: kmsan_init_arg(size_t n)
366: {
367: msan_lwp_t *lwp;
368: uint8_t *arg;
369:
370: if (__predict_false(!kmsan_enabled))
371: return;
372: lwp = curlwp->l_kmsan;
373: arg = lwp->tls[lwp->ctx].param;
374: __builtin_memset(arg, 0, n);
375: }
376:
377: void
378: kmsan_init_ret(size_t n)
379: {
380: msan_lwp_t *lwp;
381: uint8_t *arg;
382:
383: if (__predict_false(!kmsan_enabled))
384: return;
385: lwp = curlwp->l_kmsan;
386: arg = lwp->tls[lwp->ctx].retval;
387: __builtin_memset(arg, 0, n);
388: }
389:
390: static void
391: kmsan_check_arg(size_t size, const char *hook)
392: {
393: msan_lwp_t *lwp;
394: uint8_t *arg;
395: size_t i;
396:
397: if (__predict_false(!kmsan_enabled))
398: return;
399: lwp = curlwp->l_kmsan;
400: arg = lwp->tls[lwp->ctx].param;
401:
402: for (i = 0; i < size; i++) {
403: if (__predict_true(arg[i] == 0))
404: continue;
405: kmsan_report_hook((const char *)arg + i, size, i, hook);
406: break;
407: }
408: }
409:
410: void
411: kmsan_lwp_alloc(struct lwp *l)
412: {
413: msan_lwp_t *lwp;
414:
415: kmsan_init_arg(sizeof(size_t) + sizeof(km_flag_t));
416: lwp = kmem_zalloc(sizeof(msan_lwp_t), KM_SLEEP);
417: lwp->ctx = 1;
418:
419: l->l_kmsan = lwp;
420: }
421:
422: void
423: kmsan_lwp_free(struct lwp *l)
424: {
425: kmsan_init_arg(sizeof(void *) + sizeof(size_t));
426: kmem_free(l->l_kmsan, sizeof(msan_lwp_t));
427: }
428:
429: void kmsan_intr_enter(void);
430: void kmsan_intr_leave(void);
431: void kmsan_softint(struct lwp *);
432:
433: void
434: kmsan_intr_enter(void)
435: {
436: msan_lwp_t *lwp;
437:
438: if (__predict_false(!kmsan_enabled))
439: return;
440: lwp = curlwp->l_kmsan;
441:
442: lwp->ctx++;
443: if (__predict_false(lwp->ctx >= MSAN_NCONTEXT)) {
444: kmsan_enabled = false;
445: panic("%s: lwp->ctx = %zu", __func__, lwp->ctx);
446: }
447:
448: kmsan_init_arg(sizeof(void *));
449: }
450:
451: void
452: kmsan_intr_leave(void)
453: {
454: msan_lwp_t *lwp;
455:
456: if (__predict_false(!kmsan_enabled))
457: return;
458: lwp = curlwp->l_kmsan;
459:
460: if (__predict_false(lwp->ctx == 0)) {
461: kmsan_enabled = false;
462: panic("%s: lwp->ctx = %zu", __func__, lwp->ctx);
463: }
464: lwp->ctx--;
465: }
466:
467: void
468: kmsan_softint(struct lwp *l)
469: {
470: kmsan_init_arg(sizeof(lwp_t *) + sizeof(int));
471: }
472:
473: /* -------------------------------------------------------------------------- */
474:
475: void
476: kmsan_shadow_map(void *addr, size_t size)
477: {
478: size_t npages, i;
479: vaddr_t va;
480:
481: KASSERT((vaddr_t)addr % PAGE_SIZE == 0);
482: KASSERT(size % PAGE_SIZE == 0);
483:
484: npages = size / PAGE_SIZE;
485:
486: va = (vaddr_t)kmsan_md_addr_to_shad(addr);
487: for (i = 0; i < npages; i++) {
488: kmsan_md_shadow_map_page(va + i * PAGE_SIZE);
489: }
490:
491: va = (vaddr_t)kmsan_md_addr_to_orig(addr);
492: for (i = 0; i < npages; i++) {
493: kmsan_md_shadow_map_page(va + i * PAGE_SIZE);
494: }
495: }
496:
497: void
498: kmsan_orig(void *addr, size_t size, int type, uintptr_t pc)
499: {
500: msan_orig_t orig;
501:
502: orig = kmsan_md_orig_encode(type, pc);
503: kmsan_origin_fill(addr, orig, size);
504: }
505:
506: void
507: kmsan_mark(void *addr, size_t size, uint8_t c)
508: {
509: kmsan_shadow_fill(addr, c, size);
510: }
511:
512: void
513: kmsan_check_mbuf(void *buf)
514: {
515: struct mbuf *m = buf;
516:
517: do {
518: kmsan_shadow_check(mtod(m, void *), m->m_len, "if_transmit");
519: } while ((m = m->m_next) != NULL);
520: }
521:
522: void
523: kmsan_check_buf(void *buf)
524: {
525: buf_t *bp = buf;
526:
527: kmsan_shadow_check(bp->b_data, bp->b_bcount, "bwrite");
528: }
529:
530: void
531: kmsan_init(void *stack)
532: {
533: /* MD initialization. */
534: kmsan_md_init();
535:
536: /* Map the stack. */
537: kmsan_shadow_map(stack, USPACE);
538:
539: /* Initialize the TLS for curlwp. */
540: msan_lwp0.ctx = 1;
541: curlwp->l_kmsan = &msan_lwp0;
542:
543: /* Now officially enabled. */
544: kmsan_enabled = true;
545: }
546:
547: /* -------------------------------------------------------------------------- */
548:
549: msan_meta_t __msan_metadata_ptr_for_load_n(void *, size_t);
550: msan_meta_t __msan_metadata_ptr_for_store_n(void *, size_t);
551:
552: msan_meta_t __msan_metadata_ptr_for_load_n(void *addr, size_t size)
553: {
554: return kmsan_meta_get(addr, size);
555: }
556:
557: msan_meta_t __msan_metadata_ptr_for_store_n(void *addr, size_t size)
558: {
559: return kmsan_meta_get(addr, size);
560: }
561:
562: #define MSAN_META_FUNC(size) \
563: msan_meta_t __msan_metadata_ptr_for_load_##size(void *); \
564: msan_meta_t __msan_metadata_ptr_for_load_##size(void *addr) \
565: { \
566: return kmsan_meta_get(addr, size); \
567: } \
568: msan_meta_t __msan_metadata_ptr_for_store_##size(void *); \
569: msan_meta_t __msan_metadata_ptr_for_store_##size(void *addr) \
570: { \
571: return kmsan_meta_get(addr, size); \
572: }
573:
574: MSAN_META_FUNC(1)
575: MSAN_META_FUNC(2)
576: MSAN_META_FUNC(4)
577: MSAN_META_FUNC(8)
578:
579: void __msan_instrument_asm_store(void *, size_t);
580: msan_orig_t __msan_chain_origin(msan_orig_t);
581: void __msan_poison_alloca(void *, uint64_t, char *);
582: void __msan_unpoison_alloca(void *, uint64_t);
583: void __msan_warning(msan_orig_t);
584: msan_tls_t *__msan_get_context_state(void);
585:
586: void __msan_instrument_asm_store(void *addr, size_t size)
587: {
588: kmsan_shadow_fill(addr, KMSAN_STATE_INITED, size);
589: }
590:
591: msan_orig_t __msan_chain_origin(msan_orig_t origin)
592: {
593: return origin;
594: }
595:
596: void __msan_poison_alloca(void *addr, uint64_t size, char *descr)
597: {
598: msan_orig_t orig;
599:
600: orig = kmsan_md_orig_encode(KMSAN_TYPE_STACK, (uintptr_t)descr);
601: kmsan_origin_fill(addr, orig, size);
602: kmsan_shadow_fill(addr, KMSAN_STATE_UNINIT, size);
603: }
604:
605: void __msan_unpoison_alloca(void *addr, uint64_t size)
606: {
607: kmsan_shadow_fill(addr, KMSAN_STATE_INITED, size);
608: }
609:
610: void __msan_warning(msan_orig_t origin)
611: {
612: if (__predict_false(!kmsan_enabled))
613: return;
614: kmsan_report_inline(origin, __RET_ADDR);
615: }
616:
617: msan_tls_t *__msan_get_context_state(void)
618: {
619: msan_lwp_t *lwp;
620:
621: if (__predict_false(!kmsan_enabled))
622: return &dummy_tls;
623: lwp = curlwp->l_kmsan;
624:
625: return &lwp->tls[lwp->ctx];
626: }
627:
628: /* -------------------------------------------------------------------------- */
629:
630: /*
631: * Function hooks. Mostly ASM functions which need KMSAN wrappers to handle
632: * initialized areas properly.
633: */
634:
635: void *kmsan_memcpy(void *dst, const void *src, size_t len)
636: {
637: /* No kmsan_check_arg, because inlined. */
638: kmsan_init_ret(sizeof(void *));
639: if (__predict_true(len != 0)) {
640: kmsan_meta_copy(dst, src, len);
641: }
642: return __builtin_memcpy(dst, src, len);
643: }
644:
645: int
646: kmsan_memcmp(const void *b1, const void *b2, size_t len)
647: {
648: const uint8_t *_b1 = b1, *_b2 = b2;
649: size_t i;
650:
651: kmsan_check_arg(sizeof(b1) + sizeof(b2) + sizeof(len), "memcmp");
652: kmsan_init_ret(sizeof(int));
653:
654: for (i = 0; i < len; i++) {
655: if (*_b1 != *_b2) {
656: kmsan_shadow_check(b1, i + 1, "memcmp");
657: kmsan_shadow_check(b2, i + 1, "memcmp");
658: return *_b1 - *_b2;
659: }
660: _b1++, _b2++;
661: }
662:
663: return 0;
664: }
665:
666: void *kmsan_memset(void *dst, int c, size_t len)
667: {
668: /* No kmsan_check_arg, because inlined. */
669: kmsan_shadow_fill(dst, KMSAN_STATE_INITED, len);
670: kmsan_init_ret(sizeof(void *));
671: return __builtin_memset(dst, c, len);
672: }
673:
674: void *kmsan_memmove(void *dst, const void *src, size_t len)
675: {
676: /* No kmsan_check_arg, because inlined. */
677: if (__predict_true(len != 0)) {
678: kmsan_meta_copy(dst, src, len);
679: }
680: kmsan_init_ret(sizeof(void *));
681: return __builtin_memmove(dst, src, len);
682: }
683:
684: __strong_alias(__msan_memcpy, kmsan_memcpy)
685: __strong_alias(__msan_memset, kmsan_memset)
686: __strong_alias(__msan_memmove, kmsan_memmove)
687:
688: char *
689: kmsan_strcpy(char *dst, const char *src)
690: {
691: const char *_src = src;
692: char *_dst = dst;
693: size_t len = 0;
694:
695: kmsan_check_arg(sizeof(dst) + sizeof(src), "strcpy");
696:
697: while (1) {
698: len++;
699: *dst = *src;
700: if (*src == '\0')
701: break;
702: src++, dst++;
703: }
704:
705: kmsan_shadow_check(_src, len, "strcpy");
706: kmsan_shadow_fill(_dst, KMSAN_STATE_INITED, len);
707: kmsan_init_ret(sizeof(char *));
708: return _dst;
709: }
710:
711: int
712: kmsan_strcmp(const char *s1, const char *s2)
713: {
714: const char *_s1 = s1, *_s2 = s2;
715: size_t len = 0;
716:
717: kmsan_check_arg(sizeof(s1) + sizeof(s2), "strcmp");
718: kmsan_init_ret(sizeof(int));
719:
720: while (1) {
721: len++;
722: if (*s1 != *s2)
723: break;
724: if (*s1 == '\0') {
725: kmsan_shadow_check(_s1, len, "strcmp");
726: kmsan_shadow_check(_s2, len, "strcmp");
727: return 0;
728: }
729: s1++, s2++;
730: }
731:
732: kmsan_shadow_check(_s1, len, "strcmp");
733: kmsan_shadow_check(_s2, len, "strcmp");
734:
735: return (*(const unsigned char *)s1 - *(const unsigned char *)s2);
736: }
737:
738: size_t
739: kmsan_strlen(const char *str)
740: {
741: const char *s;
742:
743: kmsan_check_arg(sizeof(str), "strlen");
744:
745: s = str;
746: while (1) {
747: if (*s == '\0')
748: break;
749: s++;
750: }
751:
752: kmsan_shadow_check(str, (size_t)(s - str) + 1, "strlen");
753: kmsan_init_ret(sizeof(size_t));
754: return (s - str);
755: }
756:
757: #undef kcopy
758: #undef copystr
759: #undef copyin
760: #undef copyout
761: #undef copyinstr
762: #undef copyoutstr
763:
764: int kmsan_kcopy(const void *, void *, size_t);
765: int kmsan_copystr(const void *, void *, size_t, size_t *);
766: int kmsan_copyin(const void *, void *, size_t);
767: int kmsan_copyout(const void *, void *, size_t);
768: int kmsan_copyinstr(const void *, void *, size_t, size_t *);
769: int kmsan_copyoutstr(const void *, void *, size_t, size_t *);
770:
771: int kcopy(const void *, void *, size_t);
772: int copystr(const void *, void *, size_t, size_t *);
773: int copyin(const void *, void *, size_t);
774: int copyout(const void *, void *, size_t);
775: int copyinstr(const void *, void *, size_t, size_t *);
776: int copyoutstr(const void *, void *, size_t, size_t *);
777:
778: int
779: kmsan_kcopy(const void *src, void *dst, size_t len)
780: {
781: kmsan_check_arg(sizeof(src) + sizeof(dst) + sizeof(len), "kcopy");
782: if (__predict_true(len != 0)) {
783: kmsan_meta_copy(dst, src, len);
784: }
785: kmsan_init_ret(sizeof(int));
786: return kcopy(src, dst, len);
787: }
788:
789: int
790: kmsan_copystr(const void *kfaddr, void *kdaddr, size_t len, size_t *done)
791: {
792: size_t _done;
793: int ret;
794:
795: kmsan_check_arg(sizeof(kfaddr) + sizeof(kdaddr) +
796: sizeof(len) + sizeof(done), "copystr");
797: ret = copystr(kfaddr, kdaddr, len, &_done);
798: if (ret == 0)
799: kmsan_meta_copy(kdaddr, kfaddr, _done);
800: if (done != NULL) {
801: *done = _done;
802: kmsan_shadow_fill(done, KMSAN_STATE_INITED, sizeof(size_t));
803: }
804: kmsan_init_ret(sizeof(int));
805:
806: return ret;
807: }
808:
809: int
810: kmsan_copyin(const void *uaddr, void *kaddr, size_t len)
811: {
812: int ret;
813:
814: kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + sizeof(len), "copyin");
815: ret = copyin(uaddr, kaddr, len);
816: if (ret == 0)
817: kmsan_shadow_fill(kaddr, KMSAN_STATE_INITED, len);
818: kmsan_init_ret(sizeof(int));
819:
820: return ret;
821: }
822:
823: int
824: kmsan_copyout(const void *kaddr, void *uaddr, size_t len)
825: {
826: kmsan_check_arg(sizeof(kaddr) + sizeof(uaddr) + sizeof(len), "copyout");
827: kmsan_shadow_check(kaddr, len, "copyout");
828: kmsan_init_ret(sizeof(int));
829: return copyout(kaddr, uaddr, len);
830: }
831:
832: int
833: kmsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
834: {
835: size_t _done;
836: int ret;
837:
838: kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) +
839: sizeof(len) + sizeof(done), "copyinstr");
840: ret = copyinstr(uaddr, kaddr, len, &_done);
841: if (ret == 0)
842: kmsan_shadow_fill(kaddr, KMSAN_STATE_INITED, _done);
843: if (done != NULL) {
844: *done = _done;
845: kmsan_shadow_fill(done, KMSAN_STATE_INITED, sizeof(size_t));
846: }
847: kmsan_init_ret(sizeof(int));
848:
849: return ret;
850: }
851:
852: int
853: kmsan_copyoutstr(const void *kaddr, void *uaddr, size_t len, size_t *done)
854: {
855: size_t _done;
856: int ret;
857:
858: kmsan_check_arg(sizeof(kaddr) + sizeof(uaddr) +
859: sizeof(len) + sizeof(done), "copyoutstr");
860: ret = copyoutstr(kaddr, uaddr, len, &_done);
861: kmsan_shadow_check(kaddr, _done, "copyoutstr");
862: if (done != NULL) {
863: *done = _done;
864: kmsan_shadow_fill(done, KMSAN_STATE_INITED, sizeof(size_t));
865: }
866: kmsan_init_ret(sizeof(int));
867:
868: return ret;
869: }
870:
871: /* -------------------------------------------------------------------------- */
872:
1.2 ! maxv 873: #undef _ucas_32
! 874: #undef _ucas_32_mp
! 875: #undef _ucas_64
! 876: #undef _ucas_64_mp
! 877: #undef _ufetch_8
! 878: #undef _ufetch_16
! 879: #undef _ufetch_32
! 880: #undef _ufetch_64
! 881: #undef _ustore_8
! 882: #undef _ustore_16
! 883: #undef _ustore_32
! 884: #undef _ustore_64
! 885:
! 886: int _ucas_32(volatile uint32_t *, uint32_t, uint32_t, uint32_t *);
! 887: int kmsan__ucas_32(volatile uint32_t *, uint32_t, uint32_t, uint32_t *);
! 888: int
! 889: kmsan__ucas_32(volatile uint32_t *uaddr, uint32_t old, uint32_t new,
! 890: uint32_t *ret)
! 891: {
! 892: int _ret;
! 893: kmsan_check_arg(sizeof(uaddr) + sizeof(old) +
! 894: sizeof(new) + sizeof(ret), "ucas_32");
! 895: _ret = _ucas_32(uaddr, old, new, ret);
! 896: if (_ret == 0)
! 897: kmsan_shadow_fill(ret, KMSAN_STATE_INITED, sizeof(*ret));
! 898: kmsan_init_ret(sizeof(int));
! 899: return _ret;
! 900: }
! 901:
! 902: #ifdef __HAVE_UCAS_MP
! 903: int _ucas_32_mp(volatile uint32_t *, uint32_t, uint32_t, uint32_t *);
! 904: int kmsan__ucas_32_mp(volatile uint32_t *, uint32_t, uint32_t, uint32_t *);
! 905: int
! 906: kmsan__ucas_32_mp(volatile uint32_t *uaddr, uint32_t old, uint32_t new,
! 907: uint32_t *ret)
! 908: {
! 909: int _ret;
! 910: kmsan_check_arg(sizeof(uaddr) + sizeof(old) +
! 911: sizeof(new) + sizeof(ret), "ucas_32_mp");
! 912: _ret = _ucas_32_mp(uaddr, old, new, ret);
! 913: if (_ret == 0)
! 914: kmsan_shadow_fill(ret, KMSAN_STATE_INITED, sizeof(*ret));
! 915: kmsan_init_ret(sizeof(int));
! 916: return _ret;
! 917: }
! 918: #endif
! 919:
! 920: #ifdef _LP64
! 921: int _ucas_64(volatile uint64_t *, uint64_t, uint64_t, uint64_t *);
! 922: int kmsan__ucas_64(volatile uint64_t *, uint64_t, uint64_t, uint64_t *);
! 923: int
! 924: kmsan__ucas_64(volatile uint64_t *uaddr, uint64_t old, uint64_t new,
! 925: uint64_t *ret)
! 926: {
! 927: int _ret;
! 928: kmsan_check_arg(sizeof(uaddr) + sizeof(old) +
! 929: sizeof(new) + sizeof(ret), "ucas_64");
! 930: _ret = _ucas_64(uaddr, old, new, ret);
! 931: if (_ret == 0)
! 932: kmsan_shadow_fill(ret, KMSAN_STATE_INITED, sizeof(*ret));
! 933: kmsan_init_ret(sizeof(int));
! 934: return _ret;
! 935: }
! 936:
! 937: #ifdef __HAVE_UCAS_MP
! 938: int _ucas_64_mp(volatile uint64_t *, uint64_t, uint64_t, uint64_t *);
! 939: int kmsan__ucas_64_mp(volatile uint64_t *, uint64_t, uint64_t, uint64_t *);
! 940: int
! 941: kmsan__ucas_64_mp(volatile uint64_t *uaddr, uint64_t old, uint64_t new,
! 942: uint64_t *ret)
! 943: {
! 944: int _ret;
! 945: kmsan_check_arg(sizeof(uaddr) + sizeof(old) +
! 946: sizeof(new) + sizeof(ret), "ucas_64_mp");
! 947: _ret = _ucas_64_mp(uaddr, old, new, ret);
! 948: if (_ret == 0)
! 949: kmsan_shadow_fill(ret, KMSAN_STATE_INITED, sizeof(*ret));
! 950: kmsan_init_ret(sizeof(int));
! 951: return _ret;
! 952: }
! 953: #endif
! 954: #endif
! 955:
! 956: int _ufetch_8(const uint8_t *, uint8_t *);
! 957: int kmsan__ufetch_8(const uint8_t *, uint8_t *);
! 958: int
! 959: kmsan__ufetch_8(const uint8_t *uaddr, uint8_t *valp)
! 960: {
! 961: int _ret;
! 962: kmsan_check_arg(sizeof(uaddr) + sizeof(valp), "ufetch_8");
! 963: _ret = _ufetch_8(uaddr, valp);
! 964: if (_ret == 0)
! 965: kmsan_shadow_fill(valp, KMSAN_STATE_INITED, sizeof(*valp));
! 966: kmsan_init_ret(sizeof(int));
! 967: return _ret;
! 968: }
! 969:
! 970: int _ufetch_16(const uint16_t *, uint16_t *);
! 971: int kmsan__ufetch_16(const uint16_t *, uint16_t *);
! 972: int
! 973: kmsan__ufetch_16(const uint16_t *uaddr, uint16_t *valp)
! 974: {
! 975: int _ret;
! 976: kmsan_check_arg(sizeof(uaddr) + sizeof(valp), "ufetch_16");
! 977: _ret = _ufetch_16(uaddr, valp);
! 978: if (_ret == 0)
! 979: kmsan_shadow_fill(valp, KMSAN_STATE_INITED, sizeof(*valp));
! 980: kmsan_init_ret(sizeof(int));
! 981: return _ret;
! 982: }
! 983:
! 984: int _ufetch_32(const uint32_t *, uint32_t *);
! 985: int kmsan__ufetch_32(const uint32_t *, uint32_t *);
! 986: int
! 987: kmsan__ufetch_32(const uint32_t *uaddr, uint32_t *valp)
! 988: {
! 989: int _ret;
! 990: kmsan_check_arg(sizeof(uaddr) + sizeof(valp), "ufetch_32");
! 991: _ret = _ufetch_32(uaddr, valp);
! 992: if (_ret == 0)
! 993: kmsan_shadow_fill(valp, KMSAN_STATE_INITED, sizeof(*valp));
! 994: kmsan_init_ret(sizeof(int));
! 995: return _ret;
! 996: }
! 997:
! 998: #ifdef _LP64
! 999: int _ufetch_64(const uint64_t *, uint64_t *);
! 1000: int kmsan__ufetch_64(const uint64_t *, uint64_t *);
! 1001: int
! 1002: kmsan__ufetch_64(const uint64_t *uaddr, uint64_t *valp)
! 1003: {
! 1004: int _ret;
! 1005: kmsan_check_arg(sizeof(uaddr) + sizeof(valp), "ufetch_64");
! 1006: _ret = _ufetch_64(uaddr, valp);
! 1007: if (_ret == 0)
! 1008: kmsan_shadow_fill(valp, KMSAN_STATE_INITED, sizeof(*valp));
! 1009: kmsan_init_ret(sizeof(int));
! 1010: return _ret;
! 1011: }
! 1012: #endif
! 1013:
! 1014: int _ustore_8(uint8_t *, uint8_t);
! 1015: int kmsan__ustore_8(uint8_t *, uint8_t);
! 1016: int
! 1017: kmsan__ustore_8(uint8_t *uaddr, uint8_t val)
! 1018: {
! 1019: kmsan_check_arg(sizeof(uaddr) + sizeof(val), "ustore_8");
! 1020: kmsan_init_ret(sizeof(int));
! 1021: return _ustore_8(uaddr, val);
! 1022: }
! 1023:
! 1024: int _ustore_16(uint16_t *, uint16_t);
! 1025: int kmsan__ustore_16(uint16_t *, uint16_t);
! 1026: int
! 1027: kmsan__ustore_16(uint16_t *uaddr, uint16_t val)
! 1028: {
! 1029: kmsan_check_arg(sizeof(uaddr) + sizeof(val), "ustore_16");
! 1030: kmsan_init_ret(sizeof(int));
! 1031: return _ustore_16(uaddr, val);
! 1032: }
! 1033:
! 1034: int _ustore_32(uint32_t *, uint32_t);
! 1035: int kmsan__ustore_32(uint32_t *, uint32_t);
! 1036: int
! 1037: kmsan__ustore_32(uint32_t *uaddr, uint32_t val)
! 1038: {
! 1039: kmsan_check_arg(sizeof(uaddr) + sizeof(val), "ustore_32");
! 1040: kmsan_init_ret(sizeof(int));
! 1041: return _ustore_32(uaddr, val);
! 1042: }
! 1043:
! 1044: #ifdef _LP64
! 1045: int _ustore_64(uint64_t *, uint64_t);
! 1046: int kmsan__ustore_64(uint64_t *, uint64_t);
! 1047: int
! 1048: kmsan__ustore_64(uint64_t *uaddr, uint64_t val)
! 1049: {
! 1050: kmsan_check_arg(sizeof(uaddr) + sizeof(val), "ustore_64");
! 1051: kmsan_init_ret(sizeof(int));
! 1052: return _ustore_64(uaddr, val);
! 1053: }
! 1054: #endif
! 1055:
! 1056: /* -------------------------------------------------------------------------- */
! 1057:
1.1 maxv 1058: #undef atomic_add_32
1059: #undef atomic_add_int
1060: #undef atomic_add_long
1061: #undef atomic_add_ptr
1062: #undef atomic_add_64
1063: #undef atomic_add_32_nv
1064: #undef atomic_add_int_nv
1065: #undef atomic_add_long_nv
1066: #undef atomic_add_ptr_nv
1067: #undef atomic_add_64_nv
1068: #undef atomic_and_32
1069: #undef atomic_and_uint
1070: #undef atomic_and_ulong
1071: #undef atomic_and_64
1072: #undef atomic_and_32_nv
1073: #undef atomic_and_uint_nv
1074: #undef atomic_and_ulong_nv
1075: #undef atomic_and_64_nv
1076: #undef atomic_or_32
1077: #undef atomic_or_uint
1078: #undef atomic_or_ulong
1079: #undef atomic_or_64
1080: #undef atomic_or_32_nv
1081: #undef atomic_or_uint_nv
1082: #undef atomic_or_ulong_nv
1083: #undef atomic_or_64_nv
1084: #undef atomic_cas_32
1085: #undef atomic_cas_uint
1086: #undef atomic_cas_ulong
1087: #undef atomic_cas_ptr
1088: #undef atomic_cas_64
1089: #undef atomic_cas_32_ni
1090: #undef atomic_cas_uint_ni
1091: #undef atomic_cas_ulong_ni
1092: #undef atomic_cas_ptr_ni
1093: #undef atomic_cas_64_ni
1094: #undef atomic_swap_32
1095: #undef atomic_swap_uint
1096: #undef atomic_swap_ulong
1097: #undef atomic_swap_ptr
1098: #undef atomic_swap_64
1099: #undef atomic_dec_32
1100: #undef atomic_dec_uint
1101: #undef atomic_dec_ulong
1102: #undef atomic_dec_ptr
1103: #undef atomic_dec_64
1104: #undef atomic_dec_32_nv
1105: #undef atomic_dec_uint_nv
1106: #undef atomic_dec_ulong_nv
1107: #undef atomic_dec_ptr_nv
1108: #undef atomic_dec_64_nv
1109: #undef atomic_inc_32
1110: #undef atomic_inc_uint
1111: #undef atomic_inc_ulong
1112: #undef atomic_inc_ptr
1113: #undef atomic_inc_64
1114: #undef atomic_inc_32_nv
1115: #undef atomic_inc_uint_nv
1116: #undef atomic_inc_ulong_nv
1117: #undef atomic_inc_ptr_nv
1118: #undef atomic_inc_64_nv
1119:
1120: #define MSAN_ATOMIC_FUNC_ADD(name, tret, targ1, targ2) \
1121: void atomic_add_##name(volatile targ1 *, targ2); \
1122: void kmsan_atomic_add_##name(volatile targ1 *, targ2); \
1123: void kmsan_atomic_add_##name(volatile targ1 *ptr, targ2 val) \
1124: { \
1125: kmsan_check_arg(sizeof(ptr) + sizeof(val), __func__); \
1126: kmsan_shadow_check((uintptr_t)ptr, sizeof(tret), __func__); \
1127: atomic_add_##name(ptr, val); \
1128: } \
1129: tret atomic_add_##name##_nv(volatile targ1 *, targ2); \
1130: tret kmsan_atomic_add_##name##_nv(volatile targ1 *, targ2); \
1131: tret kmsan_atomic_add_##name##_nv(volatile targ1 *ptr, targ2 val) \
1132: { \
1133: kmsan_check_arg(sizeof(ptr) + sizeof(val), __func__); \
1134: kmsan_shadow_check((uintptr_t)ptr, sizeof(tret), __func__); \
1135: kmsan_init_ret(sizeof(tret)); \
1136: return atomic_add_##name##_nv(ptr, val); \
1137: }
1138: #define MSAN_ATOMIC_FUNC_AND(name, tret, targ1, targ2) \
1139: void atomic_and_##name(volatile targ1 *, targ2); \
1140: void kmsan_atomic_and_##name(volatile targ1 *, targ2); \
1141: void kmsan_atomic_and_##name(volatile targ1 *ptr, targ2 val) \
1142: { \
1143: kmsan_check_arg(sizeof(ptr) + sizeof(val), __func__); \
1144: kmsan_shadow_check((uintptr_t)ptr, sizeof(tret), __func__); \
1145: atomic_and_##name(ptr, val); \
1146: } \
1147: tret atomic_and_##name##_nv(volatile targ1 *, targ2); \
1148: tret kmsan_atomic_and_##name##_nv(volatile targ1 *, targ2); \
1149: tret kmsan_atomic_and_##name##_nv(volatile targ1 *ptr, targ2 val) \
1150: { \
1151: kmsan_check_arg(sizeof(ptr) + sizeof(val), __func__); \
1152: kmsan_shadow_check((uintptr_t)ptr, sizeof(tret), __func__); \
1153: kmsan_init_ret(sizeof(tret)); \
1154: return atomic_and_##name##_nv(ptr, val); \
1155: }
1156:
1157: #define MSAN_ATOMIC_FUNC_OR(name, tret, targ1, targ2) \
1158: void atomic_or_##name(volatile targ1 *, targ2); \
1159: void kmsan_atomic_or_##name(volatile targ1 *, targ2); \
1160: void kmsan_atomic_or_##name(volatile targ1 *ptr, targ2 val) \
1161: { \
1162: kmsan_check_arg(sizeof(ptr) + sizeof(val), __func__); \
1163: kmsan_shadow_check((uintptr_t)ptr, sizeof(tret), __func__); \
1164: atomic_or_##name(ptr, val); \
1165: } \
1166: tret atomic_or_##name##_nv(volatile targ1 *, targ2); \
1167: tret kmsan_atomic_or_##name##_nv(volatile targ1 *, targ2); \
1168: tret kmsan_atomic_or_##name##_nv(volatile targ1 *ptr, targ2 val) \
1169: { \
1170: kmsan_check_arg(sizeof(ptr) + sizeof(val), __func__); \
1171: kmsan_shadow_check((uintptr_t)ptr, sizeof(tret), __func__); \
1172: kmsan_init_ret(sizeof(tret)); \
1173: return atomic_or_##name##_nv(ptr, val); \
1174: }
1175:
1176: #define MSAN_ATOMIC_FUNC_CAS(name, tret, targ1, targ2) \
1177: tret atomic_cas_##name(volatile targ1 *, targ2, targ2); \
1178: tret kmsan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \
1179: tret kmsan_atomic_cas_##name(volatile targ1 *ptr, targ2 exp, targ2 new) \
1180: { \
1181: kmsan_check_arg(sizeof(ptr) + sizeof(exp) + sizeof(new), \
1182: __func__); \
1183: kmsan_shadow_check((uintptr_t)ptr, sizeof(tret), __func__); \
1184: kmsan_init_ret(sizeof(tret)); \
1185: return atomic_cas_##name(ptr, exp, new); \
1186: } \
1187: tret atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \
1188: tret kmsan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \
1189: tret kmsan_atomic_cas_##name##_ni(volatile targ1 *ptr, targ2 exp, targ2 new) \
1190: { \
1191: kmsan_check_arg(sizeof(ptr) + sizeof(exp) + sizeof(new), \
1192: __func__); \
1193: kmsan_shadow_check((uintptr_t)ptr, sizeof(tret), __func__); \
1194: kmsan_init_ret(sizeof(tret)); \
1195: return atomic_cas_##name##_ni(ptr, exp, new); \
1196: }
1197:
1198: #define MSAN_ATOMIC_FUNC_SWAP(name, tret, targ1, targ2) \
1199: tret atomic_swap_##name(volatile targ1 *, targ2); \
1200: tret kmsan_atomic_swap_##name(volatile targ1 *, targ2); \
1201: tret kmsan_atomic_swap_##name(volatile targ1 *ptr, targ2 val) \
1202: { \
1203: kmsan_check_arg(sizeof(ptr) + sizeof(val), __func__); \
1204: kmsan_shadow_check((uintptr_t)ptr, sizeof(tret), __func__); \
1205: kmsan_init_ret(sizeof(tret)); \
1206: return atomic_swap_##name(ptr, val); \
1207: }
1208:
1209: #define MSAN_ATOMIC_FUNC_DEC(name, tret, targ1) \
1210: void atomic_dec_##name(volatile targ1 *); \
1211: void kmsan_atomic_dec_##name(volatile targ1 *); \
1212: void kmsan_atomic_dec_##name(volatile targ1 *ptr) \
1213: { \
1214: kmsan_check_arg(sizeof(ptr), __func__); \
1215: kmsan_shadow_check((uintptr_t)ptr, sizeof(tret), __func__); \
1216: atomic_dec_##name(ptr); \
1217: } \
1218: tret atomic_dec_##name##_nv(volatile targ1 *); \
1219: tret kmsan_atomic_dec_##name##_nv(volatile targ1 *); \
1220: tret kmsan_atomic_dec_##name##_nv(volatile targ1 *ptr) \
1221: { \
1222: kmsan_check_arg(sizeof(ptr), __func__); \
1223: kmsan_shadow_check((uintptr_t)ptr, sizeof(tret), __func__); \
1224: kmsan_init_ret(sizeof(tret)); \
1225: return atomic_dec_##name##_nv(ptr); \
1226: }
1227:
1228: #define MSAN_ATOMIC_FUNC_INC(name, tret, targ1) \
1229: void atomic_inc_##name(volatile targ1 *); \
1230: void kmsan_atomic_inc_##name(volatile targ1 *); \
1231: void kmsan_atomic_inc_##name(volatile targ1 *ptr) \
1232: { \
1233: kmsan_check_arg(sizeof(ptr), __func__); \
1234: kmsan_shadow_check((uintptr_t)ptr, sizeof(tret), __func__); \
1235: atomic_inc_##name(ptr); \
1236: } \
1237: tret atomic_inc_##name##_nv(volatile targ1 *); \
1238: tret kmsan_atomic_inc_##name##_nv(volatile targ1 *); \
1239: tret kmsan_atomic_inc_##name##_nv(volatile targ1 *ptr) \
1240: { \
1241: kmsan_check_arg(sizeof(ptr), __func__); \
1242: kmsan_shadow_check((uintptr_t)ptr, sizeof(tret), __func__); \
1243: kmsan_init_ret(sizeof(tret)); \
1244: return atomic_inc_##name##_nv(ptr); \
1245: }
1246:
1247: MSAN_ATOMIC_FUNC_ADD(32, uint32_t, uint32_t, int32_t);
1248: MSAN_ATOMIC_FUNC_ADD(64, uint64_t, uint64_t, int64_t);
1249: MSAN_ATOMIC_FUNC_ADD(int, unsigned int, unsigned int, int);
1250: MSAN_ATOMIC_FUNC_ADD(long, unsigned long, unsigned long, long);
1251: MSAN_ATOMIC_FUNC_ADD(ptr, void *, void, ssize_t);
1252:
1253: MSAN_ATOMIC_FUNC_AND(32, uint32_t, uint32_t, uint32_t);
1254: MSAN_ATOMIC_FUNC_AND(64, uint64_t, uint64_t, uint64_t);
1255: MSAN_ATOMIC_FUNC_AND(uint, unsigned int, unsigned int, unsigned int);
1256: MSAN_ATOMIC_FUNC_AND(ulong, unsigned long, unsigned long, unsigned long);
1257:
1258: MSAN_ATOMIC_FUNC_OR(32, uint32_t, uint32_t, uint32_t);
1259: MSAN_ATOMIC_FUNC_OR(64, uint64_t, uint64_t, uint64_t);
1260: MSAN_ATOMIC_FUNC_OR(uint, unsigned int, unsigned int, unsigned int);
1261: MSAN_ATOMIC_FUNC_OR(ulong, unsigned long, unsigned long, unsigned long);
1262:
1263: MSAN_ATOMIC_FUNC_CAS(32, uint32_t, uint32_t, uint32_t);
1264: MSAN_ATOMIC_FUNC_CAS(64, uint64_t, uint64_t, uint64_t);
1265: MSAN_ATOMIC_FUNC_CAS(uint, unsigned int, unsigned int, unsigned int);
1266: MSAN_ATOMIC_FUNC_CAS(ulong, unsigned long, unsigned long, unsigned long);
1267: MSAN_ATOMIC_FUNC_CAS(ptr, void *, void, void *);
1268:
1269: MSAN_ATOMIC_FUNC_SWAP(32, uint32_t, uint32_t, uint32_t);
1270: MSAN_ATOMIC_FUNC_SWAP(64, uint64_t, uint64_t, uint64_t);
1271: MSAN_ATOMIC_FUNC_SWAP(uint, unsigned int, unsigned int, unsigned int);
1272: MSAN_ATOMIC_FUNC_SWAP(ulong, unsigned long, unsigned long, unsigned long);
1273: MSAN_ATOMIC_FUNC_SWAP(ptr, void *, void, void *);
1274:
1275: MSAN_ATOMIC_FUNC_DEC(32, uint32_t, uint32_t)
1276: MSAN_ATOMIC_FUNC_DEC(64, uint64_t, uint64_t)
1277: MSAN_ATOMIC_FUNC_DEC(uint, unsigned int, unsigned int);
1278: MSAN_ATOMIC_FUNC_DEC(ulong, unsigned long, unsigned long);
1279: MSAN_ATOMIC_FUNC_DEC(ptr, void *, void);
1280:
1281: MSAN_ATOMIC_FUNC_INC(32, uint32_t, uint32_t)
1282: MSAN_ATOMIC_FUNC_INC(64, uint64_t, uint64_t)
1283: MSAN_ATOMIC_FUNC_INC(uint, unsigned int, unsigned int);
1284: MSAN_ATOMIC_FUNC_INC(ulong, unsigned long, unsigned long);
1285: MSAN_ATOMIC_FUNC_INC(ptr, void *, void);
1286:
1287: /* -------------------------------------------------------------------------- */
1288:
1289: #include <sys/bus.h>
1290:
1291: #undef bus_space_read_multi_1
1292: #undef bus_space_read_multi_2
1293: #undef bus_space_read_multi_4
1294: #undef bus_space_read_multi_8
1295: #undef bus_space_read_multi_stream_1
1296: #undef bus_space_read_multi_stream_2
1297: #undef bus_space_read_multi_stream_4
1298: #undef bus_space_read_multi_stream_8
1299: #undef bus_space_read_region_1
1300: #undef bus_space_read_region_2
1301: #undef bus_space_read_region_4
1302: #undef bus_space_read_region_8
1303: #undef bus_space_read_region_stream_1
1304: #undef bus_space_read_region_stream_2
1305: #undef bus_space_read_region_stream_4
1306: #undef bus_space_read_region_stream_8
1307:
1308: #define MSAN_BUS_READ_FUNC(bytes, bits) \
1309: void bus_space_read_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \
1310: bus_size_t, uint##bits##_t *, bus_size_t); \
1311: void kmsan_bus_space_read_multi_##bytes(bus_space_tag_t, \
1312: bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
1313: void kmsan_bus_space_read_multi_##bytes(bus_space_tag_t tag, \
1314: bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \
1315: bus_size_t count) \
1316: { \
1317: kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \
1318: sizeof(uint##bits##_t) * count); \
1319: bus_space_read_multi_##bytes(tag, hnd, size, buf, count); \
1320: } \
1321: void bus_space_read_multi_stream_##bytes(bus_space_tag_t, \
1322: bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
1323: void kmsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t, \
1324: bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
1325: void kmsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t tag, \
1326: bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \
1327: bus_size_t count) \
1328: { \
1329: kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \
1330: sizeof(uint##bits##_t) * count); \
1331: bus_space_read_multi_stream_##bytes(tag, hnd, size, buf, count);\
1332: } \
1333: void bus_space_read_region_##bytes(bus_space_tag_t, bus_space_handle_t, \
1334: bus_size_t, uint##bits##_t *, bus_size_t); \
1335: void kmsan_bus_space_read_region_##bytes(bus_space_tag_t, \
1336: bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
1337: void kmsan_bus_space_read_region_##bytes(bus_space_tag_t tag, \
1338: bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \
1339: bus_size_t count) \
1340: { \
1341: kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \
1342: sizeof(uint##bits##_t) * count); \
1343: bus_space_read_region_##bytes(tag, hnd, size, buf, count); \
1344: } \
1345: void bus_space_read_region_stream_##bytes(bus_space_tag_t, \
1346: bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
1347: void kmsan_bus_space_read_region_stream_##bytes(bus_space_tag_t, \
1348: bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
1349: void kmsan_bus_space_read_region_stream_##bytes(bus_space_tag_t tag, \
1350: bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \
1351: bus_size_t count) \
1352: { \
1353: kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \
1354: sizeof(uint##bits##_t) * count); \
1355: bus_space_read_region_stream_##bytes(tag, hnd, size, buf, count);\
1356: }
1357:
1358: MSAN_BUS_READ_FUNC(1, 8)
1359: MSAN_BUS_READ_FUNC(2, 16)
1360: MSAN_BUS_READ_FUNC(4, 32)
1361: MSAN_BUS_READ_FUNC(8, 64)
1362:
1363: #undef bus_space_write_multi_1
1364: #undef bus_space_write_multi_2
1365: #undef bus_space_write_multi_4
1366: #undef bus_space_write_multi_8
1367: #undef bus_space_write_multi_stream_1
1368: #undef bus_space_write_multi_stream_2
1369: #undef bus_space_write_multi_stream_4
1370: #undef bus_space_write_multi_stream_8
1371: #undef bus_space_write_region_1
1372: #undef bus_space_write_region_2
1373: #undef bus_space_write_region_4
1374: #undef bus_space_write_region_8
1375: #undef bus_space_write_region_stream_1
1376: #undef bus_space_write_region_stream_2
1377: #undef bus_space_write_region_stream_4
1378: #undef bus_space_write_region_stream_8
1379:
1380: #define MSAN_BUS_WRITE_FUNC(bytes, bits) \
1381: void bus_space_write_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \
1382: bus_size_t, const uint##bits##_t *, bus_size_t); \
1383: void kmsan_bus_space_write_multi_##bytes(bus_space_tag_t, \
1384: bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
1385: void kmsan_bus_space_write_multi_##bytes(bus_space_tag_t tag, \
1386: bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \
1387: bus_size_t count) \
1388: { \
1389: kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \
1390: "bus_space_write"); \
1391: bus_space_write_multi_##bytes(tag, hnd, size, buf, count); \
1392: } \
1393: void bus_space_write_multi_stream_##bytes(bus_space_tag_t, \
1394: bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
1395: void kmsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t, \
1396: bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
1397: void kmsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t tag, \
1398: bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \
1399: bus_size_t count) \
1400: { \
1401: kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \
1402: "bus_space_write"); \
1403: bus_space_write_multi_stream_##bytes(tag, hnd, size, buf, count);\
1404: } \
1405: void bus_space_write_region_##bytes(bus_space_tag_t, bus_space_handle_t,\
1406: bus_size_t, const uint##bits##_t *, bus_size_t); \
1407: void kmsan_bus_space_write_region_##bytes(bus_space_tag_t, \
1408: bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
1409: void kmsan_bus_space_write_region_##bytes(bus_space_tag_t tag, \
1410: bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \
1411: bus_size_t count) \
1412: { \
1413: kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \
1414: "bus_space_write"); \
1415: bus_space_write_region_##bytes(tag, hnd, size, buf, count); \
1416: } \
1417: void bus_space_write_region_stream_##bytes(bus_space_tag_t, \
1418: bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
1419: void kmsan_bus_space_write_region_stream_##bytes(bus_space_tag_t, \
1420: bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
1421: void kmsan_bus_space_write_region_stream_##bytes(bus_space_tag_t tag, \
1422: bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \
1423: bus_size_t count) \
1424: { \
1425: kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \
1426: "bus_space_write"); \
1427: bus_space_write_region_stream_##bytes(tag, hnd, size, buf, count);\
1428: }
1429:
1430: MSAN_BUS_WRITE_FUNC(1, 8)
1431: MSAN_BUS_WRITE_FUNC(2, 16)
1432: MSAN_BUS_WRITE_FUNC(4, 32)
1433: MSAN_BUS_WRITE_FUNC(8, 64)
1434:
1435: /* -------------------------------------------------------------------------- */
1436:
1437: #include <sys/mbuf.h>
1438:
1439: static void
1440: kmsan_dma_sync_linear(uint8_t *buf, bus_addr_t offset, bus_size_t len,
1441: bool init, uintptr_t pc)
1442: {
1443: if (init) {
1444: kmsan_shadow_fill(buf + offset, KMSAN_STATE_INITED, len);
1445: } else {
1446: kmsan_shadow_check(buf + offset, len, "dma_sync_linear");
1447: }
1448: }
1449:
1450: static void
1451: kmsan_dma_sync_mbuf(struct mbuf *m, bus_addr_t offset, bus_size_t len,
1452: bool init, uintptr_t pc)
1453: {
1454: bus_addr_t minlen;
1455:
1456: for (; m != NULL && len != 0; m = m->m_next) {
1457: if (offset >= m->m_len) {
1458: offset -= m->m_len;
1459: continue;
1460: }
1461:
1462: minlen = MIN(len, m->m_len - offset);
1463:
1464: if (init) {
1465: kmsan_shadow_fill(mtod(m, char *) + offset,
1466: KMSAN_STATE_INITED, minlen);
1467: } else {
1468: kmsan_shadow_check(mtod(m, char *) + offset,
1469: minlen, "dma_sync_mbuf");
1470: }
1471:
1472: offset = 0;
1473: len -= minlen;
1474: }
1475: }
1476:
1477: static void
1478: kmsan_dma_sync_uio(struct uio *uio, bus_addr_t offset, bus_size_t len,
1479: bool init, uintptr_t pc)
1480: {
1481: bus_size_t minlen, resid;
1482: struct iovec *iov;
1483: int i;
1484:
1485: if (uio->uio_vmspace != NULL)
1486: return;
1487:
1488: resid = uio->uio_resid;
1489: iov = uio->uio_iov;
1490:
1491: for (i = 0; i < uio->uio_iovcnt && resid != 0; i++) {
1492: minlen = MIN(resid, iov[i].iov_len);
1493:
1494: if (init) {
1495: kmsan_shadow_fill(iov[i].iov_base,
1496: KMSAN_STATE_INITED, minlen);
1497: } else {
1498: kmsan_shadow_check(iov[i].iov_base, minlen,
1499: "dma_sync_uio");
1500: }
1501:
1502: resid -= minlen;
1503: }
1504: }
1505:
1506: void
1507: kmsan_dma_sync(bus_dmamap_t map, bus_addr_t offset, bus_size_t len, int ops)
1508: {
1509: bool init;
1510:
1511: if ((ops & (BUS_DMASYNC_PREWRITE|BUS_DMASYNC_POSTREAD)) == 0)
1512: return;
1513: init = (ops & BUS_DMASYNC_POSTREAD) != 0;
1514:
1515: switch (map->dm_buftype) {
1516: case KMSAN_DMA_LINEAR:
1517: kmsan_dma_sync_linear(map->dm_buf, offset, len, init,
1518: __RET_ADDR);
1519: break;
1520: case KMSAN_DMA_MBUF:
1521: kmsan_dma_sync_mbuf(map->dm_buf, offset, len, init,
1522: __RET_ADDR);
1523: break;
1524: case KMSAN_DMA_UIO:
1525: kmsan_dma_sync_uio(map->dm_buf, offset, len, init,
1526: __RET_ADDR);
1527: break;
1528: case KMSAN_DMA_RAW:
1529: break;
1530: default:
1531: panic("%s: impossible", __func__);
1532: }
1533: }
1534:
1535: void
1536: kmsan_dma_load(bus_dmamap_t map, void *buf, bus_size_t buflen, int type)
1537: {
1538: map->dm_buf = buf;
1539: map->dm_buflen = buflen;
1540: map->dm_buftype = type;
1541: }
CVSweb <webmaster@jp.NetBSD.org>
![[BACK]](/icons/back.gif){kind=icon}
Return to subr_msan.c CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [cvs.NetBSD.org] / src / sys / kern