version 1.78.2.4, 2017/04/26 02:53:27 |
version 1.79, 2016/09/22 16:20:56 |
Line 48 __KERNEL_RCSID(0, "$NetBSD$"); |
|
Line 48 __KERNEL_RCSID(0, "$NetBSD$"); |
|
|
|
#include <sys/atomic.h> |
#include <sys/atomic.h> |
#include <sys/conf.h> |
#include <sys/conf.h> |
#include <sys/evcnt.h> |
|
#include <sys/file.h> |
#include <sys/file.h> |
#include <sys/kauth.h> |
#include <sys/kauth.h> |
#include <sys/kernel.h> |
#include <sys/kernel.h> |
Line 71 __KERNEL_RCSID(0, "$NetBSD$"); |
|
Line 70 __KERNEL_RCSID(0, "$NetBSD$"); |
|
static struct sysctllog *wapbl_sysctl; |
static struct sysctllog *wapbl_sysctl; |
static int wapbl_flush_disk_cache = 1; |
static int wapbl_flush_disk_cache = 1; |
static int wapbl_verbose_commit = 0; |
static int wapbl_verbose_commit = 0; |
static int wapbl_allow_dpofua = 0; /* switched off by default for now */ |
|
static int wapbl_journal_iobufs = 4; |
|
|
|
static inline size_t wapbl_space_free(size_t, off_t, off_t); |
static inline size_t wapbl_space_free(size_t, off_t, off_t); |
|
|
Line 97 static inline size_t wapbl_space_free(si |
|
Line 94 static inline size_t wapbl_space_free(si |
|
* INTERNAL DATA STRUCTURES |
* INTERNAL DATA STRUCTURES |
*/ |
*/ |
|
|
/* |
/* |
* This structure holds per-mount log information. |
* This structure holds per-mount log information. |
* |
* |
* Legend: a = atomic access only |
* Legend: a = atomic access only |
|
|
* wl_count or wl_bufs or head or tail |
* wl_count or wl_bufs or head or tail |
*/ |
*/ |
|
|
#if _KERNEL |
|
/* |
/* |
* Callback called from within the flush routine to flush any extra |
* Callback called from within the flush routine to flush any extra |
* bits. Note that flush may be skipped without calling this if |
* bits. Note that flush may be skipped without calling this if |
* there are no outstanding buffers in the transaction. |
* there are no outstanding buffers in the transaction. |
*/ |
*/ |
|
#if _KERNEL |
wapbl_flush_fn_t wl_flush; /* r */ |
wapbl_flush_fn_t wl_flush; /* r */ |
wapbl_flush_fn_t wl_flush_abort;/* r */ |
wapbl_flush_fn_t wl_flush_abort;/* r */ |
|
|
/* Event counters */ |
|
char wl_ev_group[EVCNT_STRING_MAX]; /* r */ |
|
struct evcnt wl_ev_commit; /* l */ |
|
struct evcnt wl_ev_journalwrite; /* l */ |
|
struct evcnt wl_ev_jbufs_bio_nowait; /* l */ |
|
struct evcnt wl_ev_jbufs_bio_wait; /* l */ |
|
struct evcnt wl_ev_metawrite; /* lm */ |
|
struct evcnt wl_ev_cacheflush; /* l */ |
|
#endif |
#endif |
|
|
size_t wl_bufbytes; /* m: Byte count of pages in wl_bufs */ |
size_t wl_bufbytes; /* m: Byte count of pages in wl_bufs */ |
size_t wl_bufcount; /* m: Count of buffers in wl_bufs */ |
size_t wl_bufcount; /* m: Count of buffers in wl_bufs */ |
size_t wl_bcount; /* m: Total bcount of wl_bufs */ |
size_t wl_bcount; /* m: Total bcount of wl_bufs */ |
|
|
TAILQ_HEAD(, buf) wl_bufs; /* m: Buffers in current transaction */ |
LIST_HEAD(, buf) wl_bufs; /* m: Buffers in current transaction */ |
|
|
kcondvar_t wl_reclaimable_cv; /* m (obviously) */ |
kcondvar_t wl_reclaimable_cv; /* m (obviously) */ |
size_t wl_reclaimable_bytes; /* m: Amount of space available for |
size_t wl_reclaimable_bytes; /* m: Amount of space available for |
|
|
int wl_brperjblock; /* r Block records per journal block */ |
int wl_brperjblock; /* r Block records per journal block */ |
#endif |
#endif |
|
|
TAILQ_HEAD(, wapbl_dealloc) wl_dealloclist; /* lm: list head */ |
daddr_t *wl_deallocblks;/* lm: address of block */ |
int wl_dealloccnt; /* lm: total count */ |
int *wl_dealloclens; /* lm: size of block */ |
int wl_dealloclim; /* r: max count */ |
int wl_dealloccnt; /* lm: total count */ |
|
int wl_dealloclim; /* l: max count */ |
|
|
/* hashtable of inode numbers for allocated but unlinked inodes */ |
/* hashtable of inode numbers for allocated but unlinked inodes */ |
/* synch ??? */ |
/* synch ??? */ |
|
|
SIMPLEQ_HEAD(, wapbl_entry) wl_entries; /* On disk transaction |
SIMPLEQ_HEAD(, wapbl_entry) wl_entries; /* On disk transaction |
accounting */ |
accounting */ |
|
|
/* buffers for wapbl_buffered_write() */ |
u_char *wl_buffer; /* l: buffer for wapbl_buffered_write() */ |
TAILQ_HEAD(, buf) wl_iobufs; /* l: Free or filling bufs */ |
daddr_t wl_buffer_dblk; /* l: buffer disk block address */ |
TAILQ_HEAD(, buf) wl_iobufs_busy; /* l: In-transit bufs */ |
size_t wl_buffer_used; /* l: buffer current use */ |
|
|
int wl_dkcache; /* r: disk cache flags */ |
|
#define WAPBL_USE_FUA(wl) \ |
|
(wapbl_allow_dpofua && ISSET((wl)->wl_dkcache, DKCACHE_FUA)) |
|
#define WAPBL_JFLAGS(wl) \ |
|
(WAPBL_USE_FUA(wl) ? (wl)->wl_jwrite_flags : 0) |
|
#define WAPBL_MFLAGS(wl) \ |
|
(WAPBL_USE_FUA(wl) ? (wl)->wl_mwrite_flags : 0) |
|
int wl_jwrite_flags; /* r: journal write flags */ |
|
int wl_mwrite_flags; /* r: metadata write flags */ |
|
}; |
}; |
|
|
#ifdef WAPBL_DEBUG_PRINT |
#ifdef WAPBL_DEBUG_PRINT |
Line 271 static inline size_t wapbl_space_used(si |
|
Line 250 static inline size_t wapbl_space_used(si |
|
#ifdef _KERNEL |
#ifdef _KERNEL |
|
|
static struct pool wapbl_entry_pool; |
static struct pool wapbl_entry_pool; |
static struct pool wapbl_dealloc_pool; |
|
|
|
#define WAPBL_INODETRK_SIZE 83 |
#define WAPBL_INODETRK_SIZE 83 |
static int wapbl_ino_pool_refcount; |
static int wapbl_ino_pool_refcount; |
Line 289 static struct wapbl_ino *wapbl_inodetrk_ |
|
Line 267 static struct wapbl_ino *wapbl_inodetrk_ |
|
static size_t wapbl_transaction_len(struct wapbl *wl); |
static size_t wapbl_transaction_len(struct wapbl *wl); |
static inline size_t wapbl_transaction_inodes_len(struct wapbl *wl); |
static inline size_t wapbl_transaction_inodes_len(struct wapbl *wl); |
|
|
static void wapbl_deallocation_free(struct wapbl *, struct wapbl_dealloc *, |
|
bool); |
|
|
|
static void wapbl_evcnt_init(struct wapbl *); |
|
static void wapbl_evcnt_free(struct wapbl *); |
|
|
|
static void wapbl_dkcache_init(struct wapbl *); |
|
|
|
#if 0 |
#if 0 |
int wapbl_replay_verify(struct wapbl_replay *, struct vnode *); |
int wapbl_replay_verify(struct wapbl_replay *, struct vnode *); |
#endif |
#endif |
Line 351 wapbl_sysctl_init(void) |
|
Line 321 wapbl_sysctl_init(void) |
|
SYSCTL_DESCR("show time and size of wapbl log commits"), |
SYSCTL_DESCR("show time and size of wapbl log commits"), |
NULL, 0, &wapbl_verbose_commit, 0, |
NULL, 0, &wapbl_verbose_commit, 0, |
CTL_CREATE, CTL_EOL); |
CTL_CREATE, CTL_EOL); |
if (rv) |
|
return rv; |
|
|
|
rv = sysctl_createv(&wapbl_sysctl, 0, &rnode, &cnode, |
|
CTLFLAG_PERMANENT|CTLFLAG_READWRITE, |
|
CTLTYPE_INT, "allow_dpofua", |
|
SYSCTL_DESCR("allow use of FUA/DPO instead of cash flush if available"), |
|
NULL, 0, &wapbl_allow_dpofua, 0, |
|
CTL_CREATE, CTL_EOL); |
|
if (rv) |
|
return rv; |
|
|
|
rv = sysctl_createv(&wapbl_sysctl, 0, &rnode, &cnode, |
|
CTLFLAG_PERMANENT|CTLFLAG_READWRITE, |
|
CTLTYPE_INT, "journal_iobufs", |
|
SYSCTL_DESCR("count of bufs used for journal I/O (max async count)"), |
|
NULL, 0, &wapbl_journal_iobufs, 0, |
|
CTL_CREATE, CTL_EOL); |
|
if (rv) |
|
return rv; |
|
|
|
return rv; |
return rv; |
} |
} |
|
|
Line 381 wapbl_init(void) |
|
Line 330 wapbl_init(void) |
|
|
|
pool_init(&wapbl_entry_pool, sizeof(struct wapbl_entry), 0, 0, 0, |
pool_init(&wapbl_entry_pool, sizeof(struct wapbl_entry), 0, 0, 0, |
"wapblentrypl", &pool_allocator_kmem, IPL_VM); |
"wapblentrypl", &pool_allocator_kmem, IPL_VM); |
pool_init(&wapbl_dealloc_pool, sizeof(struct wapbl_dealloc), 0, 0, 0, |
|
"wapbldealloc", &pool_allocator_nointr, IPL_NONE); |
|
|
|
wapbl_sysctl_init(); |
wapbl_sysctl_init(); |
} |
} |
Line 394 wapbl_fini(void) |
|
Line 341 wapbl_fini(void) |
|
if (wapbl_sysctl != NULL) |
if (wapbl_sysctl != NULL) |
sysctl_teardown(&wapbl_sysctl); |
sysctl_teardown(&wapbl_sysctl); |
|
|
pool_destroy(&wapbl_dealloc_pool); |
|
pool_destroy(&wapbl_entry_pool); |
pool_destroy(&wapbl_entry_pool); |
|
|
return 0; |
return 0; |
} |
} |
|
|
static void |
|
wapbl_evcnt_init(struct wapbl *wl) |
|
{ |
|
snprintf(wl->wl_ev_group, sizeof(wl->wl_ev_group), |
|
"wapbl fsid 0x%x/0x%x", |
|
wl->wl_mount->mnt_stat.f_fsidx.__fsid_val[0], |
|
wl->wl_mount->mnt_stat.f_fsidx.__fsid_val[1] |
|
); |
|
|
|
evcnt_attach_dynamic(&wl->wl_ev_commit, EVCNT_TYPE_MISC, |
|
NULL, wl->wl_ev_group, "commit"); |
|
evcnt_attach_dynamic(&wl->wl_ev_journalwrite, EVCNT_TYPE_MISC, |
|
NULL, wl->wl_ev_group, "journal sync block write"); |
|
evcnt_attach_dynamic(&wl->wl_ev_jbufs_bio_nowait, EVCNT_TYPE_MISC, |
|
NULL, wl->wl_ev_group, "journal I/O bufs no wait"); |
|
evcnt_attach_dynamic(&wl->wl_ev_jbufs_bio_wait, EVCNT_TYPE_MISC, |
|
NULL, wl->wl_ev_group, "journal I/O bufs biowait"); |
|
evcnt_attach_dynamic(&wl->wl_ev_metawrite, EVCNT_TYPE_MISC, |
|
NULL, wl->wl_ev_group, "metadata finished block write"); |
|
evcnt_attach_dynamic(&wl->wl_ev_cacheflush, EVCNT_TYPE_MISC, |
|
NULL, wl->wl_ev_group, "cache flush"); |
|
} |
|
|
|
static void |
|
wapbl_evcnt_free(struct wapbl *wl) |
|
{ |
|
evcnt_detach(&wl->wl_ev_commit); |
|
evcnt_detach(&wl->wl_ev_journalwrite); |
|
evcnt_detach(&wl->wl_ev_jbufs_bio_nowait); |
|
evcnt_detach(&wl->wl_ev_jbufs_bio_wait); |
|
evcnt_detach(&wl->wl_ev_metawrite); |
|
evcnt_detach(&wl->wl_ev_cacheflush); |
|
} |
|
|
|
static void |
|
wapbl_dkcache_init(struct wapbl *wl) |
|
{ |
|
int error; |
|
|
|
/* Get disk cache flags */ |
|
error = VOP_IOCTL(wl->wl_devvp, DIOCGCACHE, &wl->wl_dkcache, |
|
FWRITE, FSCRED); |
|
if (error) { |
|
/* behave as if there was a write cache */ |
|
wl->wl_dkcache = DKCACHE_WRITE; |
|
} |
|
|
|
/* Use FUA instead of cache flush if available */ |
|
if (ISSET(wl->wl_dkcache, DKCACHE_FUA)) { |
|
wl->wl_jwrite_flags |= B_MEDIA_FUA; |
|
wl->wl_mwrite_flags |= B_MEDIA_FUA; |
|
} |
|
|
|
/* Use DPO for journal writes if available */ |
|
if (ISSET(wl->wl_dkcache, DKCACHE_DPO)) |
|
wl->wl_jwrite_flags |= B_MEDIA_DPO; |
|
} |
|
|
|
static int |
static int |
wapbl_start_flush_inodes(struct wapbl *wl, struct wapbl_replay *wr) |
wapbl_start_flush_inodes(struct wapbl *wl, struct wapbl_replay *wr) |
{ |
{ |
Line 487 wapbl_start_flush_inodes(struct wapbl *w |
|
Line 375 wapbl_start_flush_inodes(struct wapbl *w |
|
wr->wr_inodes[i].wr_imode); |
wr->wr_inodes[i].wr_imode); |
|
|
/* Make sure new transaction won't overwrite old inodes list */ |
/* Make sure new transaction won't overwrite old inodes list */ |
KDASSERT(wapbl_transaction_len(wl) <= |
KDASSERT(wapbl_transaction_len(wl) <= |
wapbl_space_free(wl->wl_circ_size, wr->wr_inodeshead, |
wapbl_space_free(wl->wl_circ_size, wr->wr_inodeshead, |
wr->wr_inodestail)); |
wr->wr_inodestail)); |
|
|
Line 560 wapbl_start(struct wapbl ** wlp, struct |
|
Line 448 wapbl_start(struct wapbl ** wlp, struct |
|
rw_init(&wl->wl_rwlock); |
rw_init(&wl->wl_rwlock); |
mutex_init(&wl->wl_mtx, MUTEX_DEFAULT, IPL_NONE); |
mutex_init(&wl->wl_mtx, MUTEX_DEFAULT, IPL_NONE); |
cv_init(&wl->wl_reclaimable_cv, "wapblrec"); |
cv_init(&wl->wl_reclaimable_cv, "wapblrec"); |
TAILQ_INIT(&wl->wl_bufs); |
LIST_INIT(&wl->wl_bufs); |
SIMPLEQ_INIT(&wl->wl_entries); |
SIMPLEQ_INIT(&wl->wl_entries); |
|
|
wl->wl_logvp = vp; |
wl->wl_logvp = vp; |
Line 614 wapbl_start(struct wapbl ** wlp, struct |
|
Line 502 wapbl_start(struct wapbl ** wlp, struct |
|
/* XXX fix actual number of buffers reserved per filesystem. */ |
/* XXX fix actual number of buffers reserved per filesystem. */ |
wl->wl_bufcount_max = (nbuf / 2) * 1024; |
wl->wl_bufcount_max = (nbuf / 2) * 1024; |
|
|
|
/* Calculate number of blocks described in a blocklist header */ |
wl->wl_brperjblock = ((1<<wl->wl_log_dev_bshift) |
wl->wl_brperjblock = ((1<<wl->wl_log_dev_bshift) |
- offsetof(struct wapbl_wc_blocklist, wc_blocks)) / |
- offsetof(struct wapbl_wc_blocklist, wc_blocks)) / |
sizeof(((struct wapbl_wc_blocklist *)0)->wc_blocks[0]); |
sizeof(((struct wapbl_wc_blocklist *)0)->wc_blocks[0]); |
Line 621 wapbl_start(struct wapbl ** wlp, struct |
|
Line 510 wapbl_start(struct wapbl ** wlp, struct |
|
|
|
/* XXX tie this into resource estimation */ |
/* XXX tie this into resource estimation */ |
wl->wl_dealloclim = wl->wl_bufbytes_max / mp->mnt_stat.f_bsize / 2; |
wl->wl_dealloclim = wl->wl_bufbytes_max / mp->mnt_stat.f_bsize / 2; |
TAILQ_INIT(&wl->wl_dealloclist); |
|
|
wl->wl_deallocblks = wapbl_alloc(sizeof(*wl->wl_deallocblks) * |
|
wl->wl_dealloclim); |
|
wl->wl_dealloclens = wapbl_alloc(sizeof(*wl->wl_dealloclens) * |
|
wl->wl_dealloclim); |
|
|
wapbl_inodetrk_init(wl, WAPBL_INODETRK_SIZE); |
wl->wl_buffer = wapbl_alloc(MAXPHYS); |
|
wl->wl_buffer_used = 0; |
wapbl_evcnt_init(wl); |
|
|
|
wapbl_dkcache_init(wl); |
wapbl_inodetrk_init(wl, WAPBL_INODETRK_SIZE); |
|
|
/* Initialize the commit header */ |
/* Initialize the commit header */ |
{ |
{ |
Line 645 wapbl_start(struct wapbl ** wlp, struct |
|
Line 537 wapbl_start(struct wapbl ** wlp, struct |
|
wl->wl_wc_scratch = wapbl_alloc(len); |
wl->wl_wc_scratch = wapbl_alloc(len); |
} |
} |
|
|
TAILQ_INIT(&wl->wl_iobufs); |
|
TAILQ_INIT(&wl->wl_iobufs_busy); |
|
for (int i = 0; i < wapbl_journal_iobufs; i++) { |
|
struct buf *bp; |
|
|
|
if ((bp = geteblk(MAXPHYS)) == NULL) |
|
goto errout; |
|
|
|
mutex_enter(&bufcache_lock); |
|
mutex_enter(devvp->v_interlock); |
|
bgetvp(devvp, bp); |
|
mutex_exit(devvp->v_interlock); |
|
mutex_exit(&bufcache_lock); |
|
|
|
bp->b_dev = devvp->v_rdev; |
|
|
|
TAILQ_INSERT_TAIL(&wl->wl_iobufs, bp, b_wapbllist); |
|
} |
|
|
|
/* |
/* |
* if there was an existing set of unlinked but |
* if there was an existing set of unlinked but |
* allocated inodes, preserve it in the new |
* allocated inodes, preserve it in the new |
Line 690 wapbl_start(struct wapbl ** wlp, struct |
|
Line 563 wapbl_start(struct wapbl ** wlp, struct |
|
wapbl_discard(wl); |
wapbl_discard(wl); |
wapbl_free(wl->wl_wc_scratch, wl->wl_wc_header->wc_len); |
wapbl_free(wl->wl_wc_scratch, wl->wl_wc_header->wc_len); |
wapbl_free(wl->wl_wc_header, wl->wl_wc_header->wc_len); |
wapbl_free(wl->wl_wc_header, wl->wl_wc_header->wc_len); |
while (!TAILQ_EMPTY(&wl->wl_iobufs)) { |
wapbl_free(wl->wl_deallocblks, |
struct buf *bp; |
sizeof(*wl->wl_deallocblks) * wl->wl_dealloclim); |
|
wapbl_free(wl->wl_dealloclens, |
bp = TAILQ_FIRST(&wl->wl_iobufs); |
sizeof(*wl->wl_dealloclens) * wl->wl_dealloclim); |
TAILQ_REMOVE(&wl->wl_iobufs, bp, b_wapbllist); |
wapbl_free(wl->wl_buffer, MAXPHYS); |
brelse(bp, BC_INVAL); |
|
} |
|
wapbl_inodetrk_free(wl); |
wapbl_inodetrk_free(wl); |
wapbl_free(wl, sizeof(*wl)); |
wapbl_free(wl, sizeof(*wl)); |
|
|
|
|
wapbl_discard(struct wapbl *wl) |
wapbl_discard(struct wapbl *wl) |
{ |
{ |
struct wapbl_entry *we; |
struct wapbl_entry *we; |
struct wapbl_dealloc *wd; |
|
struct buf *bp; |
struct buf *bp; |
int i; |
int i; |
|
|
Line 721 wapbl_discard(struct wapbl *wl) |
|
Line 591 wapbl_discard(struct wapbl *wl) |
|
* if we want to call flush from inside a transaction |
* if we want to call flush from inside a transaction |
*/ |
*/ |
rw_enter(&wl->wl_rwlock, RW_WRITER); |
rw_enter(&wl->wl_rwlock, RW_WRITER); |
wl->wl_flush(wl->wl_mount, TAILQ_FIRST(&wl->wl_dealloclist)); |
wl->wl_flush(wl->wl_mount, wl->wl_deallocblks, wl->wl_dealloclens, |
|
wl->wl_dealloccnt); |
|
|
#ifdef WAPBL_DEBUG_PRINT |
#ifdef WAPBL_DEBUG_PRINT |
{ |
{ |
Line 790 wapbl_discard(struct wapbl *wl) |
|
Line 661 wapbl_discard(struct wapbl *wl) |
|
*/ |
*/ |
mutex_enter(&bufcache_lock); |
mutex_enter(&bufcache_lock); |
mutex_enter(&wl->wl_mtx); |
mutex_enter(&wl->wl_mtx); |
while ((bp = TAILQ_FIRST(&wl->wl_bufs)) != NULL) { |
while ((bp = LIST_FIRST(&wl->wl_bufs)) != NULL) { |
if (bbusy(bp, 0, 0, &wl->wl_mtx) == 0) { |
if (bbusy(bp, 0, 0, &wl->wl_mtx) == 0) { |
/* |
/* |
* The buffer will be unlocked and |
* The buffer will be unlocked and |
Line 823 wapbl_discard(struct wapbl *wl) |
|
Line 694 wapbl_discard(struct wapbl *wl) |
|
} |
} |
|
|
/* Discard list of deallocs */ |
/* Discard list of deallocs */ |
while ((wd = TAILQ_FIRST(&wl->wl_dealloclist)) != NULL) |
wl->wl_dealloccnt = 0; |
wapbl_deallocation_free(wl, wd, true); |
|
|
|
/* XXX should we clear wl_reserved_bytes? */ |
/* XXX should we clear wl_reserved_bytes? */ |
|
|
KASSERT(wl->wl_bufbytes == 0); |
KASSERT(wl->wl_bufbytes == 0); |
KASSERT(wl->wl_bcount == 0); |
KASSERT(wl->wl_bcount == 0); |
KASSERT(wl->wl_bufcount == 0); |
KASSERT(wl->wl_bufcount == 0); |
KASSERT(TAILQ_EMPTY(&wl->wl_bufs)); |
KASSERT(LIST_EMPTY(&wl->wl_bufs)); |
KASSERT(SIMPLEQ_EMPTY(&wl->wl_entries)); |
KASSERT(SIMPLEQ_EMPTY(&wl->wl_entries)); |
KASSERT(wl->wl_inohashcnt == 0); |
KASSERT(wl->wl_inohashcnt == 0); |
KASSERT(TAILQ_EMPTY(&wl->wl_dealloclist)); |
|
KASSERT(wl->wl_dealloccnt == 0); |
|
|
|
rw_exit(&wl->wl_rwlock); |
rw_exit(&wl->wl_rwlock); |
} |
} |
Line 866 wapbl_stop(struct wapbl *wl, int force) |
|
Line 733 wapbl_stop(struct wapbl *wl, int force) |
|
KASSERT(wl->wl_bufbytes == 0); |
KASSERT(wl->wl_bufbytes == 0); |
KASSERT(wl->wl_bcount == 0); |
KASSERT(wl->wl_bcount == 0); |
KASSERT(wl->wl_bufcount == 0); |
KASSERT(wl->wl_bufcount == 0); |
KASSERT(TAILQ_EMPTY(&wl->wl_bufs)); |
KASSERT(LIST_EMPTY(&wl->wl_bufs)); |
KASSERT(wl->wl_dealloccnt == 0); |
KASSERT(wl->wl_dealloccnt == 0); |
KASSERT(SIMPLEQ_EMPTY(&wl->wl_entries)); |
KASSERT(SIMPLEQ_EMPTY(&wl->wl_entries)); |
KASSERT(wl->wl_inohashcnt == 0); |
KASSERT(wl->wl_inohashcnt == 0); |
KASSERT(TAILQ_EMPTY(&wl->wl_dealloclist)); |
|
KASSERT(wl->wl_dealloccnt == 0); |
|
KASSERT(TAILQ_EMPTY(&wl->wl_iobufs_busy)); |
|
|
|
wapbl_free(wl->wl_wc_scratch, wl->wl_wc_header->wc_len); |
wapbl_free(wl->wl_wc_scratch, wl->wl_wc_header->wc_len); |
wapbl_free(wl->wl_wc_header, wl->wl_wc_header->wc_len); |
wapbl_free(wl->wl_wc_header, wl->wl_wc_header->wc_len); |
while (!TAILQ_EMPTY(&wl->wl_iobufs)) { |
wapbl_free(wl->wl_deallocblks, |
struct buf *bp; |
sizeof(*wl->wl_deallocblks) * wl->wl_dealloclim); |
|
wapbl_free(wl->wl_dealloclens, |
bp = TAILQ_FIRST(&wl->wl_iobufs); |
sizeof(*wl->wl_dealloclens) * wl->wl_dealloclim); |
TAILQ_REMOVE(&wl->wl_iobufs, bp, b_wapbllist); |
wapbl_free(wl->wl_buffer, MAXPHYS); |
brelse(bp, BC_INVAL); |
|
} |
|
wapbl_inodetrk_free(wl); |
wapbl_inodetrk_free(wl); |
|
|
wapbl_evcnt_free(wl); |
|
|
|
cv_destroy(&wl->wl_reclaimable_cv); |
cv_destroy(&wl->wl_reclaimable_cv); |
mutex_destroy(&wl->wl_mtx); |
mutex_destroy(&wl->wl_mtx); |
rw_destroy(&wl->wl_rwlock); |
rw_destroy(&wl->wl_rwlock); |
Line 900 wapbl_stop(struct wapbl *wl, int force) |
|
Line 760 wapbl_stop(struct wapbl *wl, int force) |
|
* Unbuffered disk I/O |
* Unbuffered disk I/O |
*/ |
*/ |
|
|
static void |
static int |
wapbl_doio_accounting(struct vnode *devvp, int flags) |
wapbl_doio(void *data, size_t len, struct vnode *devvp, daddr_t pbn, int flags) |
{ |
{ |
struct pstats *pstats = curlwp->l_proc->p_stats; |
struct pstats *pstats = curlwp->l_proc->p_stats; |
|
struct buf *bp; |
|
int error; |
|
|
|
KASSERT((flags & ~(B_WRITE | B_READ)) == 0); |
|
KASSERT(devvp->v_type == VBLK); |
|
|
if ((flags & (B_WRITE | B_READ)) == B_WRITE) { |
if ((flags & (B_WRITE | B_READ)) == B_WRITE) { |
mutex_enter(devvp->v_interlock); |
mutex_enter(devvp->v_interlock); |
Line 914 wapbl_doio_accounting(struct vnode *devv |
|
Line 779 wapbl_doio_accounting(struct vnode *devv |
|
pstats->p_ru.ru_inblock++; |
pstats->p_ru.ru_inblock++; |
} |
} |
|
|
} |
|
|
|
static int |
|
wapbl_doio(void *data, size_t len, struct vnode *devvp, daddr_t pbn, int flags) |
|
{ |
|
struct buf *bp; |
|
int error; |
|
|
|
KASSERT(devvp->v_type == VBLK); |
|
|
|
wapbl_doio_accounting(devvp, flags); |
|
|
|
bp = getiobuf(devvp, true); |
bp = getiobuf(devvp, true); |
bp->b_flags = flags; |
bp->b_flags = flags; |
bp->b_cflags = BC_BUSY; /* mandatory, asserted by biowait() */ |
bp->b_cflags = BC_BUSY; /* silly & dubious */ |
bp->b_dev = devvp->v_rdev; |
bp->b_dev = devvp->v_rdev; |
bp->b_data = data; |
bp->b_data = data; |
bp->b_bufsize = bp->b_resid = bp->b_bcount = len; |
bp->b_bufsize = bp->b_resid = bp->b_bcount = len; |
Line 990 wapbl_read(void *data, size_t len, struc |
|
Line 843 wapbl_read(void *data, size_t len, struc |
|
*/ |
*/ |
|
|
/* |
/* |
* wapbl_buffered_write_async(wl, bp) |
|
* |
|
* Send buffer for asynchronous write. |
|
*/ |
|
static void |
|
wapbl_buffered_write_async(struct wapbl *wl, struct buf *bp) |
|
{ |
|
wapbl_doio_accounting(wl->wl_devvp, bp->b_flags); |
|
|
|
KASSERT(TAILQ_FIRST(&wl->wl_iobufs) == bp); |
|
TAILQ_REMOVE(&wl->wl_iobufs, bp, b_wapbllist); |
|
|
|
bp->b_flags = B_WRITE | WAPBL_JFLAGS(wl); |
|
bp->b_cflags = BC_BUSY; /* mandatory, asserted by biowait() */ |
|
bp->b_oflags = 0; |
|
bp->b_bcount = bp->b_resid; |
|
BIO_SETPRIO(bp, BPRIO_TIMECRITICAL); |
|
|
|
VOP_STRATEGY(wl->wl_devvp, bp); |
|
|
|
wl->wl_ev_journalwrite.ev_count++; |
|
|
|
TAILQ_INSERT_TAIL(&wl->wl_iobufs_busy, bp, b_wapbllist); |
|
} |
|
|
|
/* |
|
* wapbl_buffered_flush(wl) |
* wapbl_buffered_flush(wl) |
* |
* |
* Flush any buffered writes from wapbl_buffered_write. |
* Flush any buffered writes from wapbl_buffered_write. |
*/ |
*/ |
static int |
static int |
wapbl_buffered_flush(struct wapbl *wl, bool full) |
wapbl_buffered_flush(struct wapbl *wl) |
{ |
{ |
int error = 0; |
int error; |
struct buf *bp, *bnext; |
|
bool only_done = true, found = false; |
|
|
|
/* if there is outstanding buffered write, send it now */ |
|
if ((bp = TAILQ_FIRST(&wl->wl_iobufs)) && bp->b_resid > 0) |
|
wapbl_buffered_write_async(wl, bp); |
|
|
|
/* wait for I/O to complete */ |
|
again: |
|
TAILQ_FOREACH_SAFE(bp, &wl->wl_iobufs_busy, b_wapbllist, bnext) { |
|
if (!full && only_done) { |
|
/* skip unfinished */ |
|
if (!ISSET(bp->b_oflags, BO_DONE)) |
|
continue; |
|
} |
|
|
|
if (ISSET(bp->b_oflags, BO_DONE)) |
|
wl->wl_ev_jbufs_bio_nowait.ev_count++; |
|
else |
|
wl->wl_ev_jbufs_bio_wait.ev_count++; |
|
|
|
TAILQ_REMOVE(&wl->wl_iobufs_busy, bp, b_wapbllist); |
|
error = biowait(bp); |
|
|
|
/* reset for reuse */ |
|
bp->b_blkno = bp->b_resid = 0; |
|
TAILQ_INSERT_TAIL(&wl->wl_iobufs, bp, b_wapbllist); |
|
found = true; |
|
|
|
if (!full) |
if (wl->wl_buffer_used == 0) |
break; |
return 0; |
} |
|
|
|
if (!found && only_done && !TAILQ_EMPTY(&wl->wl_iobufs_busy)) { |
error = wapbl_doio(wl->wl_buffer, wl->wl_buffer_used, |
only_done = false; |
wl->wl_devvp, wl->wl_buffer_dblk, B_WRITE); |
goto again; |
wl->wl_buffer_used = 0; |
} |
|
|
|
return error; |
return error; |
} |
} |
|
|
static int |
static int |
wapbl_buffered_write(void *data, size_t len, struct wapbl *wl, daddr_t pbn) |
wapbl_buffered_write(void *data, size_t len, struct wapbl *wl, daddr_t pbn) |
{ |
{ |
|
int error; |
size_t resid; |
size_t resid; |
struct buf *bp; |
|
|
|
again: |
|
bp = TAILQ_FIRST(&wl->wl_iobufs); |
|
|
|
if (bp == NULL) { |
|
/* No more buffers, wait for any previous I/O to finish. */ |
|
wapbl_buffered_flush(wl, false); |
|
|
|
bp = TAILQ_FIRST(&wl->wl_iobufs); |
|
KASSERT(bp != NULL); |
|
} |
|
|
|
/* |
/* |
* If not adjacent to buffered data flush first. Disk block |
* If not adjacent to buffered data flush first. Disk block |
* address is always valid for non-empty buffer. |
* address is always valid for non-empty buffer. |
*/ |
*/ |
if ((bp->b_resid > 0 && pbn != bp->b_blkno + btodb(bp->b_resid))) { |
if (wl->wl_buffer_used > 0 && |
wapbl_buffered_write_async(wl, bp); |
pbn != wl->wl_buffer_dblk + btodb(wl->wl_buffer_used)) { |
goto again; |
error = wapbl_buffered_flush(wl); |
|
if (error) |
|
return error; |
} |
} |
|
|
/* |
/* |
* If this write goes to an empty buffer we have to |
* If this write goes to an empty buffer we have to |
* save the disk block address first. |
* save the disk block address first. |
*/ |
*/ |
if (bp->b_blkno == 0) |
if (wl->wl_buffer_used == 0) |
bp->b_blkno = pbn; |
wl->wl_buffer_dblk = pbn; |
|
|
/* |
/* |
* Remaining space so this buffer ends on a buffer size boundary. |
* Remaining space so this buffer ends on a MAXPHYS boundary. |
* |
* |
* Cannot become less or equal zero as the buffer would have been |
* Cannot become less or equal zero as the buffer would have been |
* flushed on the last call then. |
* flushed on the last call then. |
*/ |
*/ |
resid = bp->b_bufsize - dbtob(bp->b_blkno % btodb(bp->b_bufsize)) - |
resid = MAXPHYS - dbtob(wl->wl_buffer_dblk % btodb(MAXPHYS)) - |
bp->b_resid; |
wl->wl_buffer_used; |
KASSERT(resid > 0); |
KASSERT(resid > 0); |
KASSERT(dbtob(btodb(resid)) == resid); |
KASSERT(dbtob(btodb(resid)) == resid); |
|
|
if (len < resid) |
|
resid = len; |
|
|
|
memcpy((uint8_t *)bp->b_data + bp->b_resid, data, resid); |
|
bp->b_resid += resid; |
|
|
|
if (len >= resid) { |
if (len >= resid) { |
/* Just filled the buf, or data did not fit */ |
memcpy(wl->wl_buffer + wl->wl_buffer_used, data, resid); |
wapbl_buffered_write_async(wl, bp); |
wl->wl_buffer_used += resid; |
|
error = wapbl_doio(wl->wl_buffer, wl->wl_buffer_used, |
|
wl->wl_devvp, wl->wl_buffer_dblk, B_WRITE); |
data = (uint8_t *)data + resid; |
data = (uint8_t *)data + resid; |
len -= resid; |
len -= resid; |
pbn += btodb(resid); |
wl->wl_buffer_dblk = pbn + btodb(resid); |
|
wl->wl_buffer_used = 0; |
if (len > 0) |
if (error) |
goto again; |
return error; |
|
} |
|
KASSERT(len < MAXPHYS); |
|
if (len > 0) { |
|
memcpy(wl->wl_buffer + wl->wl_buffer_used, data, len); |
|
wl->wl_buffer_used += len; |
} |
} |
|
|
return 0; |
return 0; |
Line 1302 wapbl_add_buf(struct wapbl *wl, struct b |
|
Line 1087 wapbl_add_buf(struct wapbl *wl, struct b |
|
|
|
mutex_enter(&wl->wl_mtx); |
mutex_enter(&wl->wl_mtx); |
if (bp->b_flags & B_LOCKED) { |
if (bp->b_flags & B_LOCKED) { |
TAILQ_REMOVE(&wl->wl_bufs, bp, b_wapbllist); |
LIST_REMOVE(bp, b_wapbllist); |
WAPBL_PRINTF(WAPBL_PRINT_BUFFER2, |
WAPBL_PRINTF(WAPBL_PRINT_BUFFER2, |
("wapbl_add_buf thread %d.%d re-adding buf %p " |
("wapbl_add_buf thread %d.%d re-adding buf %p " |
"with %d bytes %d bcount\n", |
"with %d bytes %d bcount\n", |
Line 1320 wapbl_add_buf(struct wapbl *wl, struct b |
|
Line 1105 wapbl_add_buf(struct wapbl *wl, struct b |
|
curproc->p_pid, curlwp->l_lid, bp, bp->b_bufsize, |
curproc->p_pid, curlwp->l_lid, bp, bp->b_bufsize, |
bp->b_bcount)); |
bp->b_bcount)); |
} |
} |
TAILQ_INSERT_TAIL(&wl->wl_bufs, bp, b_wapbllist); |
LIST_INSERT_HEAD(&wl->wl_bufs, bp, b_wapbllist); |
mutex_exit(&wl->wl_mtx); |
mutex_exit(&wl->wl_mtx); |
|
|
bp->b_flags |= B_LOCKED; |
bp->b_flags |= B_LOCKED; |
Line 1358 wapbl_remove_buf_locked(struct wapbl * w |
|
Line 1143 wapbl_remove_buf_locked(struct wapbl * w |
|
wl->wl_bufcount--; |
wl->wl_bufcount--; |
KASSERT((wl->wl_bufcount == 0) == (wl->wl_bufbytes == 0)); |
KASSERT((wl->wl_bufcount == 0) == (wl->wl_bufbytes == 0)); |
KASSERT((wl->wl_bufcount == 0) == (wl->wl_bcount == 0)); |
KASSERT((wl->wl_bufcount == 0) == (wl->wl_bcount == 0)); |
TAILQ_REMOVE(&wl->wl_bufs, bp, b_wapbllist); |
LIST_REMOVE(bp, b_wapbllist); |
|
|
bp->b_flags &= ~B_LOCKED; |
bp->b_flags &= ~B_LOCKED; |
} |
} |
Line 1571 wapbl_truncate(struct wapbl *wl, size_t |
|
Line 1356 wapbl_truncate(struct wapbl *wl, size_t |
|
* the reserved bytes reserved. Watch out for discarded transactions, |
* the reserved bytes reserved. Watch out for discarded transactions, |
* which could leave more bytes reserved than are reclaimable. |
* which could leave more bytes reserved than are reclaimable. |
*/ |
*/ |
if (SIMPLEQ_EMPTY(&wl->wl_entries) && |
if (SIMPLEQ_EMPTY(&wl->wl_entries) && |
(delta >= wl->wl_reserved_bytes)) { |
(delta >= wl->wl_reserved_bytes)) { |
delta -= wl->wl_reserved_bytes; |
delta -= wl->wl_reserved_bytes; |
} |
} |
Line 1671 wapbl_biodone(struct buf *bp) |
|
Line 1456 wapbl_biodone(struct buf *bp) |
|
} |
} |
|
|
/* |
/* |
* Make sure that the buf doesn't retain the media flags, so that |
|
* e.g. wapbl_allow_fuadpo has immediate effect on any following I/O. |
|
* The flags will be set again if needed by another I/O. |
|
*/ |
|
bp->b_flags &= ~B_MEDIA_FLAGS; |
|
|
|
/* |
|
* Release the buffer here. wapbl_flush() may wait for the |
* Release the buffer here. wapbl_flush() may wait for the |
* log to become empty and we better unbusy the buffer before |
* log to become empty and we better unbusy the buffer before |
* wapbl_flush() returns. |
* wapbl_flush() returns. |
Line 1694 wapbl_biodone(struct buf *bp) |
|
Line 1472 wapbl_biodone(struct buf *bp) |
|
KASSERT(wl->wl_unsynced_bufbytes >= bufsize); |
KASSERT(wl->wl_unsynced_bufbytes >= bufsize); |
wl->wl_unsynced_bufbytes -= bufsize; |
wl->wl_unsynced_bufbytes -= bufsize; |
#endif |
#endif |
wl->wl_ev_metawrite.ev_count++; |
|
|
|
/* |
/* |
* If the current transaction can be reclaimed, start |
* If the current transaction can be reclaimed, start |
Line 1785 wapbl_flush(struct wapbl *wl, int waitfo |
|
Line 1562 wapbl_flush(struct wapbl *wl, int waitfo |
|
* if we want to call flush from inside a transaction |
* if we want to call flush from inside a transaction |
*/ |
*/ |
rw_enter(&wl->wl_rwlock, RW_WRITER); |
rw_enter(&wl->wl_rwlock, RW_WRITER); |
wl->wl_flush(wl->wl_mount, TAILQ_FIRST(&wl->wl_dealloclist)); |
wl->wl_flush(wl->wl_mount, wl->wl_deallocblks, wl->wl_dealloclens, |
|
wl->wl_dealloccnt); |
|
|
/* |
/* |
* Now that we are exclusively locked and the file system has |
* Now that we are exclusively locked and the file system has |
Line 1906 wapbl_flush(struct wapbl *wl, int waitfo |
|
Line 1684 wapbl_flush(struct wapbl *wl, int waitfo |
|
wl->wl_tail = tail; |
wl->wl_tail = tail; |
KASSERT(wl->wl_reclaimable_bytes >= delta); |
KASSERT(wl->wl_reclaimable_bytes >= delta); |
wl->wl_reclaimable_bytes -= delta; |
wl->wl_reclaimable_bytes -= delta; |
KDASSERT(wl->wl_dealloccnt == 0); |
wl->wl_dealloccnt = 0; |
#ifdef WAPBL_DEBUG_BUFBYTES |
#ifdef WAPBL_DEBUG_BUFBYTES |
wl->wl_unsynced_bufbytes += wl->wl_bufbytes; |
wl->wl_unsynced_bufbytes += wl->wl_bufbytes; |
#endif |
#endif |
Line 1921 wapbl_flush(struct wapbl *wl, int waitfo |
|
Line 1699 wapbl_flush(struct wapbl *wl, int waitfo |
|
SIMPLEQ_INSERT_TAIL(&wl->wl_entries, we, we_entries); |
SIMPLEQ_INSERT_TAIL(&wl->wl_entries, we, we_entries); |
|
|
/* |
/* |
* This flushes bufs in order than they were queued, so the LRU |
* this flushes bufs in reverse order than they were queued |
* order is preserved. |
* it shouldn't matter, but if we care we could use TAILQ instead. |
|
* XXX Note they will get put on the lru queue when they flush |
|
* so we might actually want to change this to preserve order. |
*/ |
*/ |
while ((bp = TAILQ_FIRST(&wl->wl_bufs)) != NULL) { |
while ((bp = LIST_FIRST(&wl->wl_bufs)) != NULL) { |
if (bbusy(bp, 0, 0, &wl->wl_mtx)) { |
if (bbusy(bp, 0, 0, &wl->wl_mtx)) { |
continue; |
continue; |
} |
} |
bp->b_iodone = wapbl_biodone; |
bp->b_iodone = wapbl_biodone; |
bp->b_private = we; |
bp->b_private = we; |
|
|
/* make sure the block is saved sync when FUA in use */ |
|
bp->b_flags |= WAPBL_MFLAGS(wl); |
|
|
|
bremfree(bp); |
bremfree(bp); |
wapbl_remove_buf_locked(wl, bp); |
wapbl_remove_buf_locked(wl, bp); |
mutex_exit(&wl->wl_mtx); |
mutex_exit(&wl->wl_mtx); |
Line 1958 wapbl_flush(struct wapbl *wl, int waitfo |
|
Line 1734 wapbl_flush(struct wapbl *wl, int waitfo |
|
* fully flushed and the on disk log is empty. |
* fully flushed and the on disk log is empty. |
*/ |
*/ |
if (waitfor) { |
if (waitfor) { |
error = wapbl_truncate(wl, wl->wl_circ_size - |
error = wapbl_truncate(wl, wl->wl_circ_size - |
wl->wl_reserved_bytes); |
wl->wl_reserved_bytes); |
} |
} |
|
|
out: |
out: |
if (error) { |
if (error) { |
wl->wl_flush_abort(wl->wl_mount, |
wl->wl_flush_abort(wl->wl_mount, wl->wl_deallocblks, |
TAILQ_FIRST(&wl->wl_dealloclist)); |
wl->wl_dealloclens, wl->wl_dealloccnt); |
} |
} |
|
|
#ifdef WAPBL_DEBUG_PRINT |
#ifdef WAPBL_DEBUG_PRINT |
Line 2089 wapbl_print(struct wapbl *wl, |
|
Line 1865 wapbl_print(struct wapbl *wl, |
|
if (full) { |
if (full) { |
int cnt = 0; |
int cnt = 0; |
(*pr)("bufs ="); |
(*pr)("bufs ="); |
TAILQ_FOREACH(bp, &wl->wl_bufs, b_wapbllist) { |
LIST_FOREACH(bp, &wl->wl_bufs, b_wapbllist) { |
if (!TAILQ_NEXT(bp, b_wapbllist)) { |
if (!LIST_NEXT(bp, b_wapbllist)) { |
(*pr)(" %p", bp); |
(*pr)(" %p", bp); |
} else if ((++cnt % 6) == 0) { |
} else if ((++cnt % 6) == 0) { |
(*pr)(" %p,\n\t", bp); |
(*pr)(" %p,\n\t", bp); |
Line 2102 wapbl_print(struct wapbl *wl, |
|
Line 1878 wapbl_print(struct wapbl *wl, |
|
|
|
(*pr)("dealloced blks = "); |
(*pr)("dealloced blks = "); |
{ |
{ |
struct wapbl_dealloc *wd; |
int i; |
cnt = 0; |
cnt = 0; |
TAILQ_FOREACH(wd, &wl->wl_dealloclist, wd_entries) { |
for (i = 0; i < wl->wl_dealloccnt; i++) { |
(*pr)(" %"PRId64":%d,", |
(*pr)(" %"PRId64":%d,", |
wd->wd_blkno, |
wl->wl_deallocblks[i], |
wd->wd_len); |
wl->wl_dealloclens[i]); |
if ((++cnt % 4) == 0) { |
if ((++cnt % 4) == 0) { |
(*pr)("\n\t"); |
(*pr)("\n\t"); |
} |
} |
Line 2136 wapbl_print(struct wapbl *wl, |
|
Line 1912 wapbl_print(struct wapbl *wl, |
|
} |
} |
(*pr)("\n"); |
(*pr)("\n"); |
} |
} |
|
|
(*pr)("iobufs free ="); |
|
TAILQ_FOREACH(bp, &wl->wl_iobufs, b_wapbllist) { |
|
if (!TAILQ_NEXT(bp, b_wapbllist)) { |
|
(*pr)(" %p", bp); |
|
} else if ((++cnt % 6) == 0) { |
|
(*pr)(" %p,\n\t", bp); |
|
} else { |
|
(*pr)(" %p,", bp); |
|
} |
|
} |
|
(*pr)("\n"); |
|
|
|
(*pr)("iobufs busy ="); |
|
TAILQ_FOREACH(bp, &wl->wl_iobufs_busy, b_wapbllist) { |
|
if (!TAILQ_NEXT(bp, b_wapbllist)) { |
|
(*pr)(" %p", bp); |
|
} else if ((++cnt % 6) == 0) { |
|
(*pr)(" %p,\n\t", bp); |
|
} else { |
|
(*pr)(" %p,", bp); |
|
} |
|
} |
|
(*pr)("\n"); |
|
} |
} |
} |
} |
|
|
Line 2179 wapbl_dump(struct wapbl *wl) |
|
Line 1931 wapbl_dump(struct wapbl *wl) |
|
|
|
/****************************************************************/ |
/****************************************************************/ |
|
|
int |
void |
wapbl_register_deallocation(struct wapbl *wl, daddr_t blk, int len, bool force, |
wapbl_register_deallocation(struct wapbl *wl, daddr_t blk, int len) |
void **cookiep) |
|
{ |
{ |
struct wapbl_dealloc *wd; |
|
int error = 0; |
|
|
|
wapbl_jlock_assert(wl); |
wapbl_jlock_assert(wl); |
|
|
mutex_enter(&wl->wl_mtx); |
mutex_enter(&wl->wl_mtx); |
|
/* XXX should eventually instead tie this into resource estimation */ |
|
/* |
|
* XXX this panic needs locking/mutex analysis and the |
|
* ability to cope with the failure. |
|
*/ |
|
/* XXX this XXX doesn't have enough XXX */ |
|
if (__predict_false(wl->wl_dealloccnt >= wl->wl_dealloclim)) |
|
panic("wapbl_register_deallocation: out of resources"); |
|
|
if (__predict_false(wl->wl_dealloccnt >= wl->wl_dealloclim)) { |
wl->wl_deallocblks[wl->wl_dealloccnt] = blk; |
if (!force) { |
wl->wl_dealloclens[wl->wl_dealloccnt] = len; |
error = EAGAIN; |
|
goto out; |
|
} |
|
|
|
/* |
|
* Forced registration can only be used when: |
|
* 1) the caller can't cope with failure |
|
* 2) the path can be triggered only bounded, small |
|
* times per transaction |
|
* If this is not fullfilled, and the path would be triggered |
|
* many times, this could overflow maximum transaction size |
|
* and panic later. |
|
*/ |
|
printf("%s: forced dealloc registration over limit: %d >= %d\n", |
|
wl->wl_mount->mnt_stat.f_mntonname, |
|
wl->wl_dealloccnt, wl->wl_dealloclim); |
|
} |
|
|
|
wl->wl_dealloccnt++; |
wl->wl_dealloccnt++; |
mutex_exit(&wl->wl_mtx); |
|
|
|
wd = pool_get(&wapbl_dealloc_pool, PR_WAITOK); |
|
wd->wd_blkno = blk; |
|
wd->wd_len = len; |
|
|
|
mutex_enter(&wl->wl_mtx); |
|
TAILQ_INSERT_TAIL(&wl->wl_dealloclist, wd, wd_entries); |
|
|
|
if (cookiep) |
|
*cookiep = wd; |
|
|
|
out: |
|
mutex_exit(&wl->wl_mtx); |
|
|
|
WAPBL_PRINTF(WAPBL_PRINT_ALLOC, |
WAPBL_PRINTF(WAPBL_PRINT_ALLOC, |
("wapbl_register_deallocation: blk=%"PRId64" len=%d error=%d\n", |
("wapbl_register_deallocation: blk=%"PRId64" len=%d\n", blk, len)); |
blk, len, error)); |
mutex_exit(&wl->wl_mtx); |
|
|
return error; |
|
} |
|
|
|
static void |
|
wapbl_deallocation_free(struct wapbl *wl, struct wapbl_dealloc *wd, |
|
bool locked) |
|
{ |
|
KASSERT(!locked |
|
|| rw_lock_held(&wl->wl_rwlock) || mutex_owned(&wl->wl_mtx)); |
|
|
|
if (!locked) |
|
mutex_enter(&wl->wl_mtx); |
|
|
|
TAILQ_REMOVE(&wl->wl_dealloclist, wd, wd_entries); |
|
wl->wl_dealloccnt--; |
|
|
|
if (!locked) |
|
mutex_exit(&wl->wl_mtx); |
|
|
|
pool_put(&wapbl_dealloc_pool, wd); |
|
} |
|
|
|
void |
|
wapbl_unregister_deallocation(struct wapbl *wl, void *cookie) |
|
{ |
|
KASSERT(cookie != NULL); |
|
wapbl_deallocation_free(wl, cookie, false); |
|
} |
} |
|
|
/****************************************************************/ |
/****************************************************************/ |
Line 2380 wapbl_transaction_len(struct wapbl *wl) |
|
Line 2076 wapbl_transaction_len(struct wapbl *wl) |
|
int blocklen = 1<<wl->wl_log_dev_bshift; |
int blocklen = 1<<wl->wl_log_dev_bshift; |
size_t len; |
size_t len; |
|
|
/* Calculate number of blocks described in a blocklist header */ |
|
len = wl->wl_bcount; |
len = wl->wl_bcount; |
len += howmany(wl->wl_bufcount, wl->wl_brperjblock) * blocklen; |
len += howmany(wl->wl_bufcount, wl->wl_brperjblock) * blocklen; |
len += howmany(wl->wl_dealloccnt, wl->wl_brperjblock) * blocklen; |
len += howmany(wl->wl_dealloccnt, wl->wl_brperjblock) * blocklen; |
Line 2405 wapbl_cache_sync(struct wapbl *wl, const |
|
Line 2100 wapbl_cache_sync(struct wapbl *wl, const |
|
int force = 1; |
int force = 1; |
int error; |
int error; |
|
|
/* Skip full cache sync if disabled, or when using FUA */ |
if (!wapbl_flush_disk_cache) { |
if (!wapbl_flush_disk_cache || WAPBL_USE_FUA(wl)) { |
|
return 0; |
return 0; |
} |
} |
if (verbose) { |
if (verbose) { |
Line 2430 wapbl_cache_sync(struct wapbl *wl, const |
|
Line 2124 wapbl_cache_sync(struct wapbl *wl, const |
|
msg, (uintmax_t)wl->wl_devvp->v_rdev, |
msg, (uintmax_t)wl->wl_devvp->v_rdev, |
(uintmax_t)ts.tv_sec, ts.tv_nsec); |
(uintmax_t)ts.tv_sec, ts.tv_nsec); |
} |
} |
|
|
wl->wl_ev_cacheflush.ev_count++; |
|
|
|
return error; |
return error; |
} |
} |
|
|
Line 2461 wapbl_write_commit(struct wapbl *wl, off |
|
Line 2152 wapbl_write_commit(struct wapbl *wl, off |
|
int error; |
int error; |
daddr_t pbn; |
daddr_t pbn; |
|
|
error = wapbl_buffered_flush(wl, true); |
error = wapbl_buffered_flush(wl); |
if (error) |
if (error) |
return error; |
return error; |
/* |
/* |
Line 2498 wapbl_write_commit(struct wapbl *wl, off |
|
Line 2189 wapbl_write_commit(struct wapbl *wl, off |
|
error = wapbl_buffered_write(wc, wc->wc_len, wl, pbn); |
error = wapbl_buffered_write(wc, wc->wc_len, wl, pbn); |
if (error) |
if (error) |
return error; |
return error; |
error = wapbl_buffered_flush(wl, true); |
error = wapbl_buffered_flush(wl); |
if (error) |
if (error) |
return error; |
return error; |
|
|
Line 2523 wapbl_write_commit(struct wapbl *wl, off |
|
Line 2214 wapbl_write_commit(struct wapbl *wl, off |
|
panic("wapbl_write_commit: error writing duplicate " |
panic("wapbl_write_commit: error writing duplicate " |
"log header: %d", error); |
"log header: %d", error); |
} |
} |
|
|
wl->wl_ev_commit.ev_count++; |
|
|
|
return 0; |
return 0; |
} |
} |
|
|
Line 2550 wapbl_write_blocks(struct wapbl *wl, off |
|
Line 2238 wapbl_write_blocks(struct wapbl *wl, off |
|
|
|
KASSERT(rw_write_held(&wl->wl_rwlock)); |
KASSERT(rw_write_held(&wl->wl_rwlock)); |
|
|
bp = TAILQ_FIRST(&wl->wl_bufs); |
bp = LIST_FIRST(&wl->wl_bufs); |
|
|
while (bp) { |
while (bp) { |
int cnt; |
int cnt; |
Line 2582 wapbl_write_blocks(struct wapbl *wl, off |
|
Line 2270 wapbl_write_blocks(struct wapbl *wl, off |
|
wc->wc_blocks[wc->wc_blkcount].wc_dlen = bp->b_bcount; |
wc->wc_blocks[wc->wc_blkcount].wc_dlen = bp->b_bcount; |
wc->wc_len += bp->b_bcount; |
wc->wc_len += bp->b_bcount; |
wc->wc_blkcount++; |
wc->wc_blkcount++; |
bp = TAILQ_NEXT(bp, b_wapbllist); |
bp = LIST_NEXT(bp, b_wapbllist); |
} |
} |
if (wc->wc_len % blocklen != 0) { |
if (wc->wc_len % blocklen != 0) { |
padding = blocklen - wc->wc_len % blocklen; |
padding = blocklen - wc->wc_len % blocklen; |
Line 2605 wapbl_write_blocks(struct wapbl *wl, off |
|
Line 2293 wapbl_write_blocks(struct wapbl *wl, off |
|
bp->b_bcount, &off); |
bp->b_bcount, &off); |
if (error) |
if (error) |
return error; |
return error; |
bp = TAILQ_NEXT(bp, b_wapbllist); |
bp = LIST_NEXT(bp, b_wapbllist); |
} |
} |
if (padding) { |
if (padding) { |
void *zero; |
void *zero; |
|
|
zero = wapbl_alloc(padding); |
zero = wapbl_alloc(padding); |
memset(zero, 0, padding); |
memset(zero, 0, padding); |
error = wapbl_circ_write(wl, zero, padding, &off); |
error = wapbl_circ_write(wl, zero, padding, &off); |
Line 2635 wapbl_write_revocations(struct wapbl *wl |
|
Line 2323 wapbl_write_revocations(struct wapbl *wl |
|
{ |
{ |
struct wapbl_wc_blocklist *wc = |
struct wapbl_wc_blocklist *wc = |
(struct wapbl_wc_blocklist *)wl->wl_wc_scratch; |
(struct wapbl_wc_blocklist *)wl->wl_wc_scratch; |
struct wapbl_dealloc *wd, *lwd; |
int i; |
int blocklen = 1<<wl->wl_log_dev_bshift; |
int blocklen = 1<<wl->wl_log_dev_bshift; |
off_t off = *offp; |
off_t off = *offp; |
int error; |
int error; |
|
|
KASSERT(rw_write_held(&wl->wl_rwlock)); |
|
|
|
if (wl->wl_dealloccnt == 0) |
if (wl->wl_dealloccnt == 0) |
return 0; |
return 0; |
|
|
while ((wd = TAILQ_FIRST(&wl->wl_dealloclist)) != NULL) { |
i = 0; |
|
while (i < wl->wl_dealloccnt) { |
wc->wc_type = WAPBL_WC_REVOCATIONS; |
wc->wc_type = WAPBL_WC_REVOCATIONS; |
wc->wc_len = blocklen; |
wc->wc_len = blocklen; |
wc->wc_blkcount = 0; |
wc->wc_blkcount = 0; |
while (wd && (wc->wc_blkcount < wl->wl_brperjblock)) { |
while ((i < wl->wl_dealloccnt) && (wc->wc_blkcount < wl->wl_brperjblock)) { |
wc->wc_blocks[wc->wc_blkcount].wc_daddr = |
wc->wc_blocks[wc->wc_blkcount].wc_daddr = |
wd->wd_blkno; |
wl->wl_deallocblks[i]; |
wc->wc_blocks[wc->wc_blkcount].wc_dlen = |
wc->wc_blocks[wc->wc_blkcount].wc_dlen = |
wd->wd_len; |
wl->wl_dealloclens[i]; |
wc->wc_blkcount++; |
wc->wc_blkcount++; |
|
i++; |
wd = TAILQ_NEXT(wd, wd_entries); |
|
} |
} |
WAPBL_PRINTF(WAPBL_PRINT_WRITE, |
WAPBL_PRINTF(WAPBL_PRINT_WRITE, |
("wapbl_write_revocations: len = %u off = %"PRIdMAX"\n", |
("wapbl_write_revocations: len = %u off = %"PRIdMAX"\n", |
Line 2664 wapbl_write_revocations(struct wapbl *wl |
|
Line 2350 wapbl_write_revocations(struct wapbl *wl |
|
error = wapbl_circ_write(wl, wc, blocklen, &off); |
error = wapbl_circ_write(wl, wc, blocklen, &off); |
if (error) |
if (error) |
return error; |
return error; |
|
|
/* free all successfully written deallocs */ |
|
lwd = wd; |
|
while ((wd = TAILQ_FIRST(&wl->wl_dealloclist)) != NULL) { |
|
if (wd == lwd) |
|
break; |
|
wapbl_deallocation_free(wl, wd, true); |
|
} |
|
} |
} |
*offp = off; |
*offp = off; |
return 0; |
return 0; |
Line 2729 wapbl_write_inodes(struct wapbl *wl, off |
|
Line 2407 wapbl_write_inodes(struct wapbl *wl, off |
|
if (error) |
if (error) |
return error; |
return error; |
} while (i < wl->wl_inohashcnt); |
} while (i < wl->wl_inohashcnt); |
|
|
*offp = off; |
*offp = off; |
return 0; |
return 0; |
} |
} |