/* $NetBSD: subr_devsw.c,v 1.53 2024/10/13 22:25:38 chs Exp $ */ /*- * Copyright (c) 2001, 2002, 2007, 2008 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation * by MAEKAWA Masahide , and by Andrew Doran. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* * Overview * * subr_devsw.c: registers device drivers by name and by major * number, and provides wrapper methods for performing I/O and * other tasks on device drivers, keying on the device number * (dev_t). * * When the system is built, the config(8) command generates * static tables of device drivers built into the kernel image * along with their associated methods. These are recorded in * the cdevsw0 and bdevsw0 tables. Drivers can also be added to * and removed from the system dynamically. * * Allocation * * When the system initially boots only the statically allocated * indexes (bdevsw0, cdevsw0) are used. If these overflow due to * allocation, we allocate a fixed block of memory to hold the new, * expanded index. This "fork" of the table is only ever performed * once in order to guarantee that other threads may safely access * the device tables: * * o Once a thread has a "reference" to the table via an earlier * open() call, we know that the entry in the table must exist * and so it is safe to access it. * * o Regardless of whether other threads see the old or new * pointers, they will point to a correct device switch * structure for the operation being performed. * * XXX Currently, the wrapper methods such as cdev_read() verify * that a device driver does in fact exist before calling the * associated driver method. This should be changed so that * once the device is has been referenced by a vnode (opened), * calling the other methods should be valid until that reference * is dropped. */ #include __KERNEL_RCSID(0, "$NetBSD: subr_devsw.c,v 1.53 2024/10/13 22:25:38 chs Exp $"); #ifdef _KERNEL_OPT #include "opt_dtrace.h" #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef DEVSW_DEBUG #define DPRINTF(x) printf x #else /* DEVSW_DEBUG */ #define DPRINTF(x) #endif /* DEVSW_DEBUG */ #define MAXDEVSW 512 /* the maximum of major device number */ #define BDEVSW_SIZE (sizeof(struct bdevsw *)) #define CDEVSW_SIZE (sizeof(struct cdevsw *)) #define DEVSWCONV_SIZE (sizeof(struct devsw_conv)) struct devswref { struct localcount *dr_lc; }; /* XXX bdevsw, cdevsw, max_bdevsws, and max_cdevsws should be volatile */ extern const struct bdevsw **bdevsw, *bdevsw0[]; extern const struct cdevsw **cdevsw, *cdevsw0[]; extern struct devsw_conv *devsw_conv, devsw_conv0[]; extern const int sys_bdevsws, sys_cdevsws; extern int max_bdevsws, max_cdevsws, max_devsw_convs; static struct devswref *cdevswref; static struct devswref *bdevswref; static kcondvar_t devsw_cv; static int bdevsw_attach(const struct bdevsw *, devmajor_t *); static int cdevsw_attach(const struct cdevsw *, devmajor_t *); static void devsw_detach_locked(const struct bdevsw *, const struct cdevsw *); kmutex_t device_lock; void (*biodone_vfs)(buf_t *) = (void *)nullop; /* * bdev probes */ SDT_PROBE_DEFINE6(sdt, bdev, open, acquire, "struct bdevsw *"/*bdevsw*/, "dev_t"/*dev*/, "int"/*flag*/, "int"/*devtype*/, "int"/*unit*/, "device_t"/*dv*/); SDT_PROBE_DEFINE4(sdt, bdev, open, entry, "struct bdevsw *"/*bdevsw*/, "dev_t"/*dev*/, "int"/*flag*/, "int"/*devtype*/); SDT_PROBE_DEFINE5(sdt, bdev, open, return, "struct bdevsw *"/*bdevsw*/, "dev_t"/*dev*/, "int"/*flag*/, "int"/*devtype*/, "int"/*error*/); SDT_PROBE_DEFINE6(sdt, bdev, open, release, "struct bdevsw *"/*bdevsw*/, "dev_t"/*dev*/, "int"/*flag*/, "int"/*devtype*/, "int"/*unit*/, "device_t"/*dv*/); SDT_PROBE_DEFINE4(sdt, bdev, cancel, entry, "struct bdevsw *"/*bdevsw*/, "dev_t"/*dev*/, "int"/*flag*/, "int"/*devtype*/); SDT_PROBE_DEFINE5(sdt, bdev, cancel, return, "struct bdevsw *"/*bdevsw*/, "dev_t"/*dev*/, "int"/*flag*/, "int"/*devtype*/, "int"/*error*/); SDT_PROBE_DEFINE4(sdt, bdev, close, entry, "struct bdevsw *"/*bdevsw*/, "dev_t"/*dev*/, "int"/*flag*/, "int"/*devtype*/); SDT_PROBE_DEFINE5(sdt, bdev, close, return, "struct bdevsw *"/*bdevsw*/, "dev_t"/*dev*/, "int"/*flag*/, "int"/*devtype*/, "int"/*error*/); SDT_PROBE_DEFINE3(sdt, bdev, strategy, entry, "struct bdevsw *"/*bdevsw*/, "dev_t"/*dev*/, "struct buf *"/*bp*/); SDT_PROBE_DEFINE3(sdt, bdev, strategy, return, "struct bdevsw *"/*bdevsw*/, "dev_t"/*dev*/, "struct buf *"/*bp*/); SDT_PROBE_DEFINE5(sdt, bdev, ioctl, entry, "struct bdevsw *"/*bdevsw*/, "dev_t"/*dev*/, "unsigned long"/*cmd*/, "void *"/*data*/, "int"/*flag*/); SDT_PROBE_DEFINE6(sdt, bdev, ioctl, return, "struct bdevsw *"/*bdevsw*/, "dev_t"/*dev*/, "unsigned long"/*cmd*/, "void *"/*data*/, "int"/*flag*/, "int"/*error*/); SDT_PROBE_DEFINE2(sdt, bdev, psize, entry, "struct bdevsw *"/*bdevsw*/, "dev_t"/*dev*/); SDT_PROBE_DEFINE3(sdt, bdev, psize, return, "struct bdevsw *"/*bdevsw*/, "dev_t"/*dev*/, "int"/*psize*/); SDT_PROBE_DEFINE4(sdt, bdev, discard, entry, "struct bdevsw *"/*bdevsw*/, "dev_t"/*dev*/, "off_t"/*pos*/, "off_t"/*len*/); SDT_PROBE_DEFINE5(sdt, bdev, discard, return, "struct bdevsw *"/*bdevsw*/, "dev_t"/*dev*/, "off_t"/*pos*/, "off_t"/*len*/, "int"/*error*/); /* * cdev probes */ SDT_PROBE_DEFINE6(sdt, cdev, open, acquire, "struct cdevsw *"/*cdevsw*/, "dev_t"/*dev*/, "int"/*flag*/, "int"/*devtype*/, "int"/*unit*/, "device_t"/*dv*/); SDT_PROBE_DEFINE4(sdt, cdev, open, entry, "struct cdevsw *"/*cdevsw*/, "dev_t"/*dev*/, "int"/*flag*/, "int"/*devtype*/); SDT_PROBE_DEFINE5(sdt, cdev, open, return, "struct cdevsw *"/*cdevsw*/, "dev_t"/*dev*/, "int"/*flag*/, "int"/*devtype*/, "int"/*error*/); SDT_PROBE_DEFINE6(sdt, cdev, open, release, "struct cdevsw *"/*cdevsw*/, "dev_t"/*dev*/, "int"/*flag*/, "int"/*devtype*/, "int"/*unit*/, "device_t"/*dv*/); SDT_PROBE_DEFINE4(sdt, cdev, cancel, entry, "struct cdevsw *"/*cdevsw*/, "dev_t"/*dev*/, "int"/*flag*/, "int"/*devtype*/); SDT_PROBE_DEFINE5(sdt, cdev, cancel, return, "struct cdevsw *"/*cdevsw*/, "dev_t"/*dev*/, "int"/*flag*/, "int"/*devtype*/, "int"/*error*/); SDT_PROBE_DEFINE4(sdt, cdev, close, entry, "struct cdevsw *"/*cdevsw*/, "dev_t"/*dev*/, "int"/*flag*/, "int"/*devtype*/); SDT_PROBE_DEFINE5(sdt, cdev, close, return, "struct cdevsw *"/*cdevsw*/, "dev_t"/*dev*/, "int"/*flag*/, "int"/*devtype*/, "int"/*error*/); SDT_PROBE_DEFINE4(sdt, cdev, read, entry, "struct cdevsw *"/*cdevsw*/, "dev_t"/*dev*/, "struct uio *"/*uio*/, "int"/*flag*/); SDT_PROBE_DEFINE5(sdt, cdev, read, return, "struct cdevsw *"/*cdevsw*/, "dev_t"/*dev*/, "struct uio *"/*uio*/, "int"/*flag*/, "int"/*error*/); SDT_PROBE_DEFINE4(sdt, cdev, write, entry, "struct cdevsw *"/*cdevsw*/, "dev_t"/*dev*/, "struct uio *"/*uio*/, "int"/*flag*/); SDT_PROBE_DEFINE5(sdt, cdev, write, return, "struct cdevsw *"/*cdevsw*/, "dev_t"/*dev*/, "struct uio *"/*uio*/, "int"/*flag*/, "int"/*error*/); SDT_PROBE_DEFINE5(sdt, cdev, ioctl, entry, "struct cdevsw *"/*cdevsw*/, "dev_t"/*dev*/, "unsigned long"/*cmd*/, "void *"/*data*/, "int"/*flag*/); SDT_PROBE_DEFINE6(sdt, cdev, ioctl, return, "struct cdevsw *"/*cdevsw*/, "dev_t"/*dev*/, "unsigned long"/*cmd*/, "void *"/*data*/, "int"/*flag*/, "int"/*error*/); SDT_PROBE_DEFINE4(sdt, cdev, stop, entry, "struct cdevsw *"/*cdevsw*/, "dev_t"/*dev*/, "struct tty *"/*tp*/, "int"/*flag*/); SDT_PROBE_DEFINE4(sdt, cdev, stop, return, "struct cdevsw *"/*cdevsw*/, "dev_t"/*dev*/, "struct tty *"/*tp*/, "int"/*flag*/); SDT_PROBE_DEFINE3(sdt, cdev, poll, entry, "struct cdevsw *"/*cdevsw*/, "dev_t"/*dev*/, "int"/*events*/); SDT_PROBE_DEFINE4(sdt, cdev, poll, return, "struct cdevsw *"/*cdevsw*/, "dev_t"/*dev*/, "int"/*events*/, "int"/*revents*/); SDT_PROBE_DEFINE4(sdt, cdev, mmap, entry, "struct cdevsw *"/*cdevsw*/, "dev_t"/*dev*/, "off_t"/*off*/, "int"/*flag*/); SDT_PROBE_DEFINE5(sdt, cdev, mmap, return, "struct cdevsw *"/*cdevsw*/, "dev_t"/*dev*/, "off_t"/*off*/, "int"/*flag*/, "paddr_t"/*mmapcookie*/); SDT_PROBE_DEFINE3(sdt, cdev, kqfilter, entry, "struct cdevsw *"/*cdevsw*/, "dev_t"/*dev*/, "struct knote *"/*kn*/); SDT_PROBE_DEFINE4(sdt, cdev, kqfilter, return, "struct cdevsw *"/*cdevsw*/, "dev_t"/*dev*/, "struct knote *"/*kn*/, "int"/*error*/); SDT_PROBE_DEFINE4(sdt, cdev, discard, entry, "struct cdevsw *"/*cdevsw*/, "dev_t"/*dev*/, "off_t"/*pos*/, "off_t"/*len*/); SDT_PROBE_DEFINE5(sdt, cdev, discard, return, "struct cdevsw *"/*cdevsw*/, "dev_t"/*dev*/, "off_t"/*pos*/, "off_t"/*len*/, "int"/*error*/); void devsw_init(void) { KASSERT(sys_bdevsws < MAXDEVSW - 1); KASSERT(sys_cdevsws < MAXDEVSW - 1); mutex_init(&device_lock, MUTEX_DEFAULT, IPL_NONE); cv_init(&devsw_cv, "devsw"); } int devsw_attach(const char *devname, const struct bdevsw *bdev, devmajor_t *bmajor, const struct cdevsw *cdev, devmajor_t *cmajor) { struct devsw_conv *conv; char *name; int error, i; if (devname == NULL || cdev == NULL) return EINVAL; mutex_enter(&device_lock); for (i = 0; i < max_devsw_convs; i++) { conv = &devsw_conv[i]; if (conv->d_name == NULL || strcmp(devname, conv->d_name) != 0) continue; if ((bdev != NULL) && (*bmajor < 0)) *bmajor = conv->d_bmajor; if (*cmajor < 0) *cmajor = conv->d_cmajor; if (*bmajor != conv->d_bmajor || *cmajor != conv->d_cmajor) { error = EINVAL; goto out; } if ((*bmajor >= 0 && bdev == NULL) || *cmajor < 0) { error = EINVAL; goto out; } if ((*bmajor >= 0 && bdevsw[*bmajor] != NULL) || cdevsw[*cmajor] != NULL) { error = EEXIST; goto out; } break; } /* * XXX This should allocate what it needs up front so we never * need to flail around trying to unwind. */ error = bdevsw_attach(bdev, bmajor); if (error != 0) goto out; error = cdevsw_attach(cdev, cmajor); if (error != 0) { devsw_detach_locked(bdev, NULL); goto out; } /* * If we already found a conv, we're done. Otherwise, find an * empty slot or extend the table. */ if (i < max_devsw_convs) { error = 0; goto out; } for (i = 0; i < max_devsw_convs; i++) { if (devsw_conv[i].d_name == NULL) break; } if (i == max_devsw_convs) { struct devsw_conv *newptr; int old_convs, new_convs; old_convs = max_devsw_convs; new_convs = old_convs + 1; newptr = kmem_zalloc(new_convs * DEVSWCONV_SIZE, KM_NOSLEEP); if (newptr == NULL) { devsw_detach_locked(bdev, cdev); error = ENOMEM; goto out; } newptr[old_convs].d_name = NULL; newptr[old_convs].d_bmajor = -1; newptr[old_convs].d_cmajor = -1; memcpy(newptr, devsw_conv, old_convs * DEVSWCONV_SIZE); if (devsw_conv != devsw_conv0) kmem_free(devsw_conv, old_convs * DEVSWCONV_SIZE); devsw_conv = newptr; max_devsw_convs = new_convs; } name = kmem_strdupsize(devname, NULL, KM_NOSLEEP); if (name == NULL) { devsw_detach_locked(bdev, cdev); error = ENOMEM; goto out; } devsw_conv[i].d_name = name; devsw_conv[i].d_bmajor = *bmajor; devsw_conv[i].d_cmajor = *cmajor; error = 0; out: mutex_exit(&device_lock); return error; } static int bdevsw_attach(const struct bdevsw *devsw, devmajor_t *devmajor) { const struct bdevsw **newbdevsw = NULL; struct devswref *newbdevswref = NULL; struct localcount *lc; devmajor_t bmajor; int i; KASSERT(mutex_owned(&device_lock)); if (devsw == NULL) return 0; if (*devmajor < 0) { for (bmajor = sys_bdevsws; bmajor < max_bdevsws; bmajor++) { if (bdevsw[bmajor] != NULL) continue; for (i = 0; i < max_devsw_convs; i++) { if (devsw_conv[i].d_bmajor == bmajor) break; } if (i != max_devsw_convs) continue; break; } *devmajor = bmajor; } if (*devmajor >= MAXDEVSW) { printf("%s: block majors exhausted\n", __func__); return ENOMEM; } if (bdevswref == NULL) { newbdevswref = kmem_zalloc(MAXDEVSW * sizeof(newbdevswref[0]), KM_NOSLEEP); if (newbdevswref == NULL) return ENOMEM; atomic_store_release(&bdevswref, newbdevswref); } if (*devmajor >= max_bdevsws) { KASSERT(bdevsw == bdevsw0); newbdevsw = kmem_zalloc(MAXDEVSW * sizeof(newbdevsw[0]), KM_NOSLEEP); if (newbdevsw == NULL) return ENOMEM; memcpy(newbdevsw, bdevsw, max_bdevsws * sizeof(bdevsw[0])); atomic_store_release(&bdevsw, newbdevsw); atomic_store_release(&max_bdevsws, MAXDEVSW); } if (bdevsw[*devmajor] != NULL) return EEXIST; KASSERT(bdevswref[*devmajor].dr_lc == NULL); lc = kmem_zalloc(sizeof(*lc), KM_SLEEP); localcount_init(lc); bdevswref[*devmajor].dr_lc = lc; atomic_store_release(&bdevsw[*devmajor], devsw); return 0; } static int cdevsw_attach(const struct cdevsw *devsw, devmajor_t *devmajor) { const struct cdevsw **newcdevsw = NULL; struct devswref *newcdevswref = NULL; struct localcount *lc; devmajor_t cmajor; int i; KASSERT(mutex_owned(&device_lock)); if (*devmajor < 0) { for (cmajor = sys_cdevsws; cmajor < max_cdevsws; cmajor++) { if (cdevsw[cmajor] != NULL) continue; for (i = 0; i < max_devsw_convs; i++) { if (devsw_conv[i].d_cmajor == cmajor) break; } if (i != max_devsw_convs) continue; break; } *devmajor = cmajor; } if (*devmajor >= MAXDEVSW) { printf("%s: character majors exhausted\n", __func__); return ENOMEM; } if (cdevswref == NULL) { newcdevswref = kmem_zalloc(MAXDEVSW * sizeof(newcdevswref[0]), KM_NOSLEEP); if (newcdevswref == NULL) return ENOMEM; atomic_store_release(&cdevswref, newcdevswref); } if (*devmajor >= max_cdevsws) { KASSERT(cdevsw == cdevsw0); newcdevsw = kmem_zalloc(MAXDEVSW * sizeof(newcdevsw[0]), KM_NOSLEEP); if (newcdevsw == NULL) return ENOMEM; memcpy(newcdevsw, cdevsw, max_cdevsws * sizeof(cdevsw[0])); atomic_store_release(&cdevsw, newcdevsw); atomic_store_release(&max_cdevsws, MAXDEVSW); } if (cdevsw[*devmajor] != NULL) return EEXIST; KASSERT(cdevswref[*devmajor].dr_lc == NULL); lc = kmem_zalloc(sizeof(*lc), KM_SLEEP); localcount_init(lc); cdevswref[*devmajor].dr_lc = lc; atomic_store_release(&cdevsw[*devmajor], devsw); return 0; } static void devsw_detach_locked(const struct bdevsw *bdev, const struct cdevsw *cdev) { int bi = -1, ci = -1/*XXXGCC*/, di; struct cfdriver *cd; device_t dv; KASSERT(mutex_owned(&device_lock)); /* * If this is wired to an autoconf device, make sure the device * has no more instances. No locking here because under * correct use of devsw_detach, none of this state can change * at this point. */ if (cdev != NULL && (cd = cdev->d_cfdriver) != NULL) { for (di = 0; di < cd->cd_ndevs; di++) { KASSERTMSG((dv = cd->cd_devs[di]) == NULL, "detaching character device driver %s" " still has attached unit %s", cd->cd_name, device_xname(dv)); } } if (bdev != NULL && (cd = bdev->d_cfdriver) != NULL) { for (di = 0; di < cd->cd_ndevs; di++) { KASSERTMSG((dv = cd->cd_devs[di]) == NULL, "detaching block device driver %s" " still has attached unit %s", cd->cd_name, device_xname(dv)); } } /* Prevent new references. */ if (bdev != NULL) { for (bi = 0; bi < max_bdevsws; bi++) { if (bdevsw[bi] != bdev) continue; atomic_store_relaxed(&bdevsw[bi], NULL); break; } KASSERT(bi < max_bdevsws); } if (cdev != NULL) { for (ci = 0; ci < max_cdevsws; ci++) { if (cdevsw[ci] != cdev) continue; atomic_store_relaxed(&cdevsw[ci], NULL); break; } KASSERT(ci < max_cdevsws); } if (bdev == NULL && cdev == NULL) /* XXX possible? */ return; /* * Wait for all bdevsw_lookup_acquire, cdevsw_lookup_acquire * calls to notice that the devsw is gone. * * XXX Despite the use of the pserialize_read_enter/exit API * elsewhere in this file, we use xc_barrier here instead of * pserialize_perform -- because devsw_init is too early for * pserialize_create. Either pserialize_create should be made * to work earlier, or it should be nixed altogether. Until * that is fixed, xc_barrier will serve the same purpose. */ xc_barrier(0); /* * Wait for all references to drain. It is the caller's * responsibility to ensure that at this point, there are no * extant open instances and all new d_open calls will fail. * * Note that localcount_drain may release and reacquire * device_lock. */ if (bdev != NULL) { localcount_drain(bdevswref[bi].dr_lc, &devsw_cv, &device_lock); localcount_fini(bdevswref[bi].dr_lc); kmem_free(bdevswref[bi].dr_lc, sizeof(*bdevswref[bi].dr_lc)); bdevswref[bi].dr_lc = NULL; } if (cdev != NULL) { localcount_drain(cdevswref[ci].dr_lc, &devsw_cv, &device_lock); localcount_fini(cdevswref[ci].dr_lc); kmem_free(cdevswref[ci].dr_lc, sizeof(*cdevswref[ci].dr_lc)); cdevswref[ci].dr_lc = NULL; } } void devsw_detach(const struct bdevsw *bdev, const struct cdevsw *cdev) { mutex_enter(&device_lock); devsw_detach_locked(bdev, cdev); mutex_exit(&device_lock); } /* * Look up a block device by number. * * => Caller must ensure that the device is attached. */ const struct bdevsw * bdevsw_lookup(dev_t dev) { devmajor_t bmajor; if (dev == NODEV) return NULL; bmajor = major(dev); if (bmajor < 0 || bmajor >= atomic_load_relaxed(&max_bdevsws)) return NULL; return atomic_load_consume(&bdevsw)[bmajor]; } static const struct bdevsw * bdevsw_lookup_acquire(dev_t dev, struct localcount **lcp) { devmajor_t bmajor; const struct bdevsw *bdev = NULL, *const *curbdevsw; struct devswref *curbdevswref; int s; if (dev == NODEV) return NULL; bmajor = major(dev); if (bmajor < 0) return NULL; s = pserialize_read_enter(); /* * max_bdevsws never goes down, so it is safe to rely on this * condition without any locking for the array access below. * Test sys_bdevsws first so we can avoid the memory barrier in * that case. */ if (bmajor >= sys_bdevsws && bmajor >= atomic_load_acquire(&max_bdevsws)) goto out; curbdevsw = atomic_load_consume(&bdevsw); if ((bdev = atomic_load_consume(&curbdevsw[bmajor])) == NULL) goto out; curbdevswref = atomic_load_consume(&bdevswref); if (curbdevswref == NULL) { *lcp = NULL; } else if ((*lcp = curbdevswref[bmajor].dr_lc) != NULL) { localcount_acquire(*lcp); } out: pserialize_read_exit(s); return bdev; } static void bdevsw_release(const struct bdevsw *bdev, struct localcount *lc) { if (lc == NULL) return; localcount_release(lc, &devsw_cv, &device_lock); } /* * Look up a character device by number. * * => Caller must ensure that the device is attached. */ const struct cdevsw * cdevsw_lookup(dev_t dev) { devmajor_t cmajor; if (dev == NODEV) return NULL; cmajor = major(dev); if (cmajor < 0 || cmajor >= atomic_load_relaxed(&max_cdevsws)) return NULL; return atomic_load_consume(&cdevsw)[cmajor]; } static const struct cdevsw * cdevsw_lookup_acquire(dev_t dev, struct localcount **lcp) { devmajor_t cmajor; const struct cdevsw *cdev = NULL, *const *curcdevsw; struct devswref *curcdevswref; int s; if (dev == NODEV) return NULL; cmajor = major(dev); if (cmajor < 0) return NULL; s = pserialize_read_enter(); /* * max_cdevsws never goes down, so it is safe to rely on this * condition without any locking for the array access below. * Test sys_cdevsws first so we can avoid the memory barrier in * that case. */ if (cmajor >= sys_cdevsws && cmajor >= atomic_load_acquire(&max_cdevsws)) goto out; curcdevsw = atomic_load_consume(&cdevsw); if ((cdev = atomic_load_consume(&curcdevsw[cmajor])) == NULL) goto out; curcdevswref = atomic_load_consume(&cdevswref); if (curcdevswref == NULL) { *lcp = NULL; } else if ((*lcp = curcdevswref[cmajor].dr_lc) != NULL) { localcount_acquire(*lcp); } out: pserialize_read_exit(s); return cdev; } static void cdevsw_release(const struct cdevsw *cdev, struct localcount *lc) { if (lc == NULL) return; localcount_release(lc, &devsw_cv, &device_lock); } /* * Look up a block device by reference to its operations set. * * => Caller must ensure that the device is not detached, and therefore * that the returned major is still valid when dereferenced. */ devmajor_t bdevsw_lookup_major(const struct bdevsw *bdev) { const struct bdevsw *const *curbdevsw; devmajor_t bmajor, bmax; bmax = atomic_load_acquire(&max_bdevsws); curbdevsw = atomic_load_consume(&bdevsw); for (bmajor = 0; bmajor < bmax; bmajor++) { if (atomic_load_relaxed(&curbdevsw[bmajor]) == bdev) return bmajor; } return NODEVMAJOR; } /* * Look up a character device by reference to its operations set. * * => Caller must ensure that the device is not detached, and therefore * that the returned major is still valid when dereferenced. */ devmajor_t cdevsw_lookup_major(const struct cdevsw *cdev) { const struct cdevsw *const *curcdevsw; devmajor_t cmajor, cmax; cmax = atomic_load_acquire(&max_cdevsws); curcdevsw = atomic_load_consume(&cdevsw); for (cmajor = 0; cmajor < cmax; cmajor++) { if (atomic_load_relaxed(&curcdevsw[cmajor]) == cdev) return cmajor; } return NODEVMAJOR; } /* * Convert from block major number to name. * * => Caller must ensure that the device is not detached, and therefore * that the name pointer is still valid when dereferenced. */ const char * devsw_blk2name(devmajor_t bmajor) { const char *name; devmajor_t cmajor; int i; name = NULL; cmajor = -1; mutex_enter(&device_lock); if (bmajor < 0 || bmajor >= max_bdevsws || bdevsw[bmajor] == NULL) { mutex_exit(&device_lock); return NULL; } for (i = 0; i < max_devsw_convs; i++) { if (devsw_conv[i].d_bmajor == bmajor) { cmajor = devsw_conv[i].d_cmajor; break; } } if (cmajor >= 0 && cmajor < max_cdevsws && cdevsw[cmajor] != NULL) name = devsw_conv[i].d_name; mutex_exit(&device_lock); return name; } /* * Convert char major number to device driver name. */ const char * cdevsw_getname(devmajor_t major) { const char *name; int i; name = NULL; if (major < 0) return NULL; mutex_enter(&device_lock); for (i = 0; i < max_devsw_convs; i++) { if (devsw_conv[i].d_cmajor == major) { name = devsw_conv[i].d_name; break; } } mutex_exit(&device_lock); return name; } /* * Convert block major number to device driver name. */ const char * bdevsw_getname(devmajor_t major) { const char *name; int i; name = NULL; if (major < 0) return NULL; mutex_enter(&device_lock); for (i = 0; i < max_devsw_convs; i++) { if (devsw_conv[i].d_bmajor == major) { name = devsw_conv[i].d_name; break; } } mutex_exit(&device_lock); return name; } /* * Convert from device name to block major number. * * => Caller must ensure that the device is not detached, and therefore * that the major number is still valid when dereferenced. */ devmajor_t devsw_name2blk(const char *name, char *devname, size_t devnamelen) { struct devsw_conv *conv; devmajor_t bmajor; int i; if (name == NULL) return NODEVMAJOR; mutex_enter(&device_lock); for (i = 0; i < max_devsw_convs; i++) { size_t len; conv = &devsw_conv[i]; if (conv->d_name == NULL) continue; len = strlen(conv->d_name); if (strncmp(conv->d_name, name, len) != 0) continue; if (name[len] != '\0' && !isdigit((unsigned char)name[len])) continue; bmajor = conv->d_bmajor; if (bmajor < 0 || bmajor >= max_bdevsws || bdevsw[bmajor] == NULL) break; if (devname != NULL) { #ifdef DEVSW_DEBUG if (strlen(conv->d_name) >= devnamelen) printf("%s: too short buffer\n", __func__); #endif /* DEVSW_DEBUG */ strncpy(devname, conv->d_name, devnamelen); devname[devnamelen - 1] = '\0'; } mutex_exit(&device_lock); return bmajor; } mutex_exit(&device_lock); return NODEVMAJOR; } /* * Convert from device name to char major number. * * => Caller must ensure that the device is not detached, and therefore * that the major number is still valid when dereferenced. */ devmajor_t devsw_name2chr(const char *name, char *devname, size_t devnamelen) { struct devsw_conv *conv; devmajor_t cmajor; int i; if (name == NULL) return NODEVMAJOR; mutex_enter(&device_lock); for (i = 0; i < max_devsw_convs; i++) { size_t len; conv = &devsw_conv[i]; if (conv->d_name == NULL) continue; len = strlen(conv->d_name); if (strncmp(conv->d_name, name, len) != 0) continue; if (name[len] != '\0' && !isdigit((unsigned char)name[len])) continue; cmajor = conv->d_cmajor; if (cmajor < 0 || cmajor >= max_cdevsws || cdevsw[cmajor] == NULL) break; if (devname != NULL) { #ifdef DEVSW_DEBUG if (strlen(conv->d_name) >= devnamelen) printf("%s: too short buffer", __func__); #endif /* DEVSW_DEBUG */ strncpy(devname, conv->d_name, devnamelen); devname[devnamelen - 1] = '\0'; } mutex_exit(&device_lock); return cmajor; } mutex_exit(&device_lock); return NODEVMAJOR; } /* * Convert from character dev_t to block dev_t. * * => Caller must ensure that the device is not detached, and therefore * that the major number is still valid when dereferenced. */ dev_t devsw_chr2blk(dev_t cdev) { devmajor_t bmajor, cmajor; int i; dev_t rv; cmajor = major(cdev); bmajor = NODEVMAJOR; rv = NODEV; mutex_enter(&device_lock); if (cmajor < 0 || cmajor >= max_cdevsws || cdevsw[cmajor] == NULL) { mutex_exit(&device_lock); return NODEV; } for (i = 0; i < max_devsw_convs; i++) { if (devsw_conv[i].d_cmajor == cmajor) { bmajor = devsw_conv[i].d_bmajor; break; } } if (bmajor >= 0 && bmajor < max_bdevsws && bdevsw[bmajor] != NULL) rv = makedev(bmajor, minor(cdev)); mutex_exit(&device_lock); return rv; } /* * Convert from block dev_t to character dev_t. * * => Caller must ensure that the device is not detached, and therefore * that the major number is still valid when dereferenced. */ dev_t devsw_blk2chr(dev_t bdev) { devmajor_t bmajor, cmajor; int i; dev_t rv; bmajor = major(bdev); cmajor = NODEVMAJOR; rv = NODEV; mutex_enter(&device_lock); if (bmajor < 0 || bmajor >= max_bdevsws || bdevsw[bmajor] == NULL) { mutex_exit(&device_lock); return NODEV; } for (i = 0; i < max_devsw_convs; i++) { if (devsw_conv[i].d_bmajor == bmajor) { cmajor = devsw_conv[i].d_cmajor; break; } } if (cmajor >= 0 && cmajor < max_cdevsws && cdevsw[cmajor] != NULL) rv = makedev(cmajor, minor(bdev)); mutex_exit(&device_lock); return rv; } /* * Device access methods. */ #define DEV_LOCK(d) \ if ((mpflag = (d->d_flag & D_MPSAFE)) == 0) { \ KERNEL_LOCK(1, NULL); \ } #define DEV_UNLOCK(d) \ if (mpflag == 0) { \ KERNEL_UNLOCK_ONE(NULL); \ } int bdev_open(dev_t dev, int flag, int devtype, lwp_t *l) { const struct bdevsw *d; struct localcount *lc; device_t dv = NULL/*XXXGCC*/; int unit = -1/*XXXGCC*/, rv, mpflag; d = bdevsw_lookup_acquire(dev, &lc); if (d == NULL) return ENXIO; if (d->d_devtounit) { /* * If the device node corresponds to an autoconf device * instance, acquire a reference to it so that during * d_open, device_lookup is stable. * * XXX This should also arrange to instantiate cloning * pseudo-devices if appropriate, but that requires * reviewing them all to find and verify a common * pattern. */ if ((unit = (*d->d_devtounit)(dev)) == -1) { rv = ENXIO; goto out; } if ((dv = device_lookup_acquire(d->d_cfdriver, unit)) == NULL) { rv = ENXIO; goto out; } SDT_PROBE6(sdt, bdev, open, acquire, d, dev, flag, devtype, unit, dv); } DEV_LOCK(d); SDT_PROBE4(sdt, bdev, open, entry, d, dev, flag, devtype); rv = (*d->d_open)(dev, flag, devtype, l); SDT_PROBE5(sdt, bdev, open, return, d, dev, flag, devtype, rv); DEV_UNLOCK(d); if (d->d_devtounit) { SDT_PROBE6(sdt, bdev, open, release, d, dev, flag, devtype, unit, dv); device_release(dv); } out: bdevsw_release(d, lc); return rv; } int bdev_cancel(dev_t dev, int flag, int devtype, struct lwp *l) { const struct bdevsw *d; int rv, mpflag; if ((d = bdevsw_lookup(dev)) == NULL) return ENXIO; if (d->d_cancel == NULL) return ENODEV; DEV_LOCK(d); SDT_PROBE4(sdt, bdev, cancel, entry, d, dev, flag, devtype); rv = (*d->d_cancel)(dev, flag, devtype, l); SDT_PROBE5(sdt, bdev, cancel, return, d, dev, flag, devtype, rv); DEV_UNLOCK(d); return rv; } int bdev_close(dev_t dev, int flag, int devtype, lwp_t *l) { const struct bdevsw *d; int rv, mpflag; if ((d = bdevsw_lookup(dev)) == NULL) return ENXIO; DEV_LOCK(d); SDT_PROBE4(sdt, bdev, close, entry, d, dev, flag, devtype); rv = (*d->d_close)(dev, flag, devtype, l); SDT_PROBE5(sdt, bdev, close, return, d, dev, flag, devtype, rv); DEV_UNLOCK(d); return rv; } SDT_PROVIDER_DECLARE(io); SDT_PROBE_DEFINE1(io, kernel, , start, "struct buf *"/*bp*/); void bdev_strategy(struct buf *bp) { const struct bdevsw *d; int mpflag; SDT_PROBE1(io, kernel, , start, bp); if ((d = bdevsw_lookup(bp->b_dev)) == NULL) { bp->b_error = ENXIO; bp->b_resid = bp->b_bcount; biodone_vfs(bp); /* biodone() iff vfs present */ return; } DEV_LOCK(d); SDT_PROBE3(sdt, bdev, strategy, entry, d, bp->b_dev, bp); (*d->d_strategy)(bp); SDT_PROBE3(sdt, bdev, strategy, return, d, bp->b_dev, bp); DEV_UNLOCK(d); } int bdev_ioctl(dev_t dev, u_long cmd, void *data, int flag, lwp_t *l) { const struct bdevsw *d; int rv, mpflag; if ((d = bdevsw_lookup(dev)) == NULL) return ENXIO; DEV_LOCK(d); SDT_PROBE5(sdt, bdev, ioctl, entry, d, dev, cmd, data, flag); rv = (*d->d_ioctl)(dev, cmd, data, flag, l); SDT_PROBE6(sdt, bdev, ioctl, return, d, dev, cmd, data, flag, rv); DEV_UNLOCK(d); return rv; } int bdev_dump(dev_t dev, daddr_t addr, void *data, size_t sz) { const struct bdevsw *d; int rv; /* * Dump can be called without the device open. Since it can * currently only be called with the system paused (and in a * potentially unstable state), we don't perform any locking. */ if ((d = bdevsw_lookup(dev)) == NULL) return ENXIO; /* DEV_LOCK(d); */ rv = (*d->d_dump)(dev, addr, data, sz); /* DEV_UNLOCK(d); */ return rv; } int bdev_flags(dev_t dev) { const struct bdevsw *d; if ((d = bdevsw_lookup(dev)) == NULL) return 0; return d->d_flag & ~D_TYPEMASK; } int bdev_type(dev_t dev) { const struct bdevsw *d; if ((d = bdevsw_lookup(dev)) == NULL) return D_OTHER; return d->d_flag & D_TYPEMASK; } int bdev_size(dev_t dev) { const struct bdevsw *d; int rv, mpflag = 0; if ((d = bdevsw_lookup(dev)) == NULL || d->d_psize == NULL) return -1; /* * Don't to try lock the device if we're dumping. * XXX: is there a better way to test this? */ if ((boothowto & RB_DUMP) == 0) DEV_LOCK(d); SDT_PROBE2(sdt, bdev, psize, entry, d, dev); rv = (*d->d_psize)(dev); SDT_PROBE3(sdt, bdev, psize, return, d, dev, rv); if ((boothowto & RB_DUMP) == 0) DEV_UNLOCK(d); return rv; } int bdev_discard(dev_t dev, off_t pos, off_t len) { const struct bdevsw *d; int rv, mpflag; if ((d = bdevsw_lookup(dev)) == NULL) return ENXIO; DEV_LOCK(d); SDT_PROBE4(sdt, bdev, discard, entry, d, dev, pos, len); rv = (*d->d_discard)(dev, pos, len); SDT_PROBE5(sdt, bdev, discard, return, d, dev, pos, len, rv); DEV_UNLOCK(d); return rv; } void bdev_detached(dev_t dev) { const struct bdevsw *d; device_t dv; int unit; if ((d = bdevsw_lookup(dev)) == NULL) return; if (d->d_devtounit == NULL) return; if ((unit = (*d->d_devtounit)(dev)) == -1) return; if ((dv = device_lookup(d->d_cfdriver, unit)) == NULL) return; config_detach_commit(dv); } int cdev_open(dev_t dev, int flag, int devtype, lwp_t *l) { const struct cdevsw *d; struct localcount *lc; device_t dv = NULL/*XXXGCC*/; int unit = -1/*XXXGCC*/, rv, mpflag; d = cdevsw_lookup_acquire(dev, &lc); if (d == NULL) return ENXIO; if (d->d_devtounit) { /* * If the device node corresponds to an autoconf device * instance, acquire a reference to it so that during * d_open, device_lookup is stable. * * XXX This should also arrange to instantiate cloning * pseudo-devices if appropriate, but that requires * reviewing them all to find and verify a common * pattern. */ if ((unit = (*d->d_devtounit)(dev)) == -1) { rv = ENXIO; goto out; } if ((dv = device_lookup_acquire(d->d_cfdriver, unit)) == NULL) { rv = ENXIO; goto out; } SDT_PROBE6(sdt, cdev, open, acquire, d, dev, flag, devtype, unit, dv); } DEV_LOCK(d); SDT_PROBE4(sdt, cdev, open, entry, d, dev, flag, devtype); rv = (*d->d_open)(dev, flag, devtype, l); SDT_PROBE5(sdt, cdev, open, return, d, dev, flag, devtype, rv); DEV_UNLOCK(d); if (d->d_devtounit) { SDT_PROBE6(sdt, cdev, open, release, d, dev, flag, devtype, unit, dv); device_release(dv); } out: cdevsw_release(d, lc); return rv; } int cdev_cancel(dev_t dev, int flag, int devtype, struct lwp *l) { const struct cdevsw *d; int rv, mpflag; if ((d = cdevsw_lookup(dev)) == NULL) return ENXIO; if (d->d_cancel == NULL) return ENODEV; DEV_LOCK(d); SDT_PROBE4(sdt, cdev, cancel, entry, d, dev, flag, devtype); rv = (*d->d_cancel)(dev, flag, devtype, l); SDT_PROBE5(sdt, cdev, cancel, return, d, dev, flag, devtype, rv); DEV_UNLOCK(d); return rv; } int cdev_close(dev_t dev, int flag, int devtype, lwp_t *l) { const struct cdevsw *d; int rv, mpflag; if ((d = cdevsw_lookup(dev)) == NULL) return ENXIO; DEV_LOCK(d); SDT_PROBE4(sdt, cdev, close, entry, d, dev, flag, devtype); rv = (*d->d_close)(dev, flag, devtype, l); SDT_PROBE5(sdt, cdev, close, return, d, dev, flag, devtype, rv); DEV_UNLOCK(d); return rv; } int cdev_read(dev_t dev, struct uio *uio, int flag) { const struct cdevsw *d; int rv, mpflag; if ((d = cdevsw_lookup(dev)) == NULL) return ENXIO; DEV_LOCK(d); SDT_PROBE4(sdt, cdev, read, entry, d, dev, uio, flag); rv = (*d->d_read)(dev, uio, flag); SDT_PROBE5(sdt, cdev, read, return, d, dev, uio, flag, rv); DEV_UNLOCK(d); return rv; } int cdev_write(dev_t dev, struct uio *uio, int flag) { const struct cdevsw *d; int rv, mpflag; if ((d = cdevsw_lookup(dev)) == NULL) return ENXIO; DEV_LOCK(d); SDT_PROBE4(sdt, cdev, write, entry, d, dev, uio, flag); rv = (*d->d_write)(dev, uio, flag); SDT_PROBE5(sdt, cdev, write, return, d, dev, uio, flag, rv); DEV_UNLOCK(d); return rv; } int cdev_ioctl(dev_t dev, u_long cmd, void *data, int flag, lwp_t *l) { const struct cdevsw *d; int rv, mpflag; if ((d = cdevsw_lookup(dev)) == NULL) return ENXIO; DEV_LOCK(d); SDT_PROBE5(sdt, cdev, ioctl, entry, d, dev, cmd, data, flag); rv = (*d->d_ioctl)(dev, cmd, data, flag, l); SDT_PROBE6(sdt, cdev, ioctl, return, d, dev, cmd, data, flag, rv); DEV_UNLOCK(d); return rv; } void cdev_stop(struct tty *tp, int flag) { const struct cdevsw *d; int mpflag; if ((d = cdevsw_lookup(tp->t_dev)) == NULL) return; DEV_LOCK(d); SDT_PROBE4(sdt, cdev, stop, entry, d, tp->t_dev, tp, flag); (*d->d_stop)(tp, flag); SDT_PROBE4(sdt, cdev, stop, return, d, tp->t_dev, tp, flag); DEV_UNLOCK(d); } struct tty * cdev_tty(dev_t dev) { const struct cdevsw *d; if ((d = cdevsw_lookup(dev)) == NULL) return NULL; /* XXX Check if necessary. */ if (d->d_tty == NULL) return NULL; return (*d->d_tty)(dev); } int cdev_poll(dev_t dev, int flag, lwp_t *l) { const struct cdevsw *d; int rv, mpflag; if ((d = cdevsw_lookup(dev)) == NULL) return POLLERR; DEV_LOCK(d); SDT_PROBE3(sdt, cdev, poll, entry, d, dev, flag); rv = (*d->d_poll)(dev, flag, l); SDT_PROBE4(sdt, cdev, poll, return, d, dev, flag, rv); DEV_UNLOCK(d); return rv; } paddr_t cdev_mmap(dev_t dev, off_t off, int flag) { const struct cdevsw *d; paddr_t rv; int mpflag; if ((d = cdevsw_lookup(dev)) == NULL) return (paddr_t)-1LL; DEV_LOCK(d); SDT_PROBE4(sdt, cdev, mmap, entry, d, dev, off, flag); rv = (*d->d_mmap)(dev, off, flag); SDT_PROBE5(sdt, cdev, mmap, return, d, dev, off, flag, rv); DEV_UNLOCK(d); return rv; } int cdev_kqfilter(dev_t dev, struct knote *kn) { const struct cdevsw *d; int rv, mpflag; if ((d = cdevsw_lookup(dev)) == NULL) return ENXIO; DEV_LOCK(d); SDT_PROBE3(sdt, cdev, kqfilter, entry, d, dev, kn); rv = (*d->d_kqfilter)(dev, kn); SDT_PROBE4(sdt, cdev, kqfilter, return, d, dev, kn, rv); DEV_UNLOCK(d); return rv; } int cdev_discard(dev_t dev, off_t pos, off_t len) { const struct cdevsw *d; int rv, mpflag; if ((d = cdevsw_lookup(dev)) == NULL) return ENXIO; DEV_LOCK(d); SDT_PROBE4(sdt, cdev, discard, entry, d, dev, pos, len); rv = (*d->d_discard)(dev, pos, len); SDT_PROBE5(sdt, cdev, discard, return, d, dev, pos, len, rv); DEV_UNLOCK(d); return rv; } int cdev_flags(dev_t dev) { const struct cdevsw *d; if ((d = cdevsw_lookup(dev)) == NULL) return 0; return d->d_flag & ~D_TYPEMASK; } int cdev_type(dev_t dev) { const struct cdevsw *d; if ((d = cdevsw_lookup(dev)) == NULL) return D_OTHER; return d->d_flag & D_TYPEMASK; } void cdev_detached(dev_t dev) { const struct cdevsw *d; device_t dv; int unit; if ((d = cdevsw_lookup(dev)) == NULL) return; if (d->d_devtounit == NULL) return; if ((unit = (*d->d_devtounit)(dev)) == -1) return; if ((dv = device_lookup(d->d_cfdriver, unit)) == NULL) return; config_detach_commit(dv); } /* * nommap(dev, off, prot) * * mmap routine that always fails, for non-mmappable devices. */ paddr_t nommap(dev_t dev, off_t off, int prot) { return (paddr_t)-1; } /* * dev_minor_unit(dev) * * Returns minor(dev) as an int. Intended for use with struct * bdevsw, cdevsw::d_devtounit for drivers whose /dev nodes are * implemented by reference to an autoconf instance with the minor * number. */ int dev_minor_unit(dev_t dev) { return minor(dev); }