From 7f34244a35719de4ab7b753c9dc6bc6192574e90 Mon Sep 17 00:00:00 2001 From: David Teigland Date: Apr 30 2012 19:11:10 +0000 Subject: sanlock: ASYNC flag for lockspace add and rem New SANLK_ADD_ASYNC for sanlock_add_lockspace(), and SANLK_REM_ASYNC for sanlock_rem_lockspace(). The command will finish before the delta lease operation on disk. With ASYNC, both return 0 if add/rem is successfully started. sanlock_inq_lockspace() can be used to check the status of the lockspace after add or rem is started. sanlock_add_lockspace() still returns: 0 add started (with ASYNC) or completed successfully (sync) -EEXIST lockspace already exists -EINPROGRESS add lockspace in progress -EAGAIN remove lockspace in progress -EINVAL a lockspace exists with different properties (same name but different disk locations, or different names but same disk location) sanlock_rem_lockspace() still returns: 0 remove started (with ASYNC) or completed successfully (sync) -EINPROGRESS remove lockspace in progress -ENOENT no lockspace with matching name and disk location is found sanlock_inq_lockspace() still returns: 0 lockspace exists and is not being added or removed -EINPROGRESS add or rem is in progress -ENOENT no lockspace with matching name and disk location is found Signed-off-by: David Teigland --- diff --git a/src/cmd.c b/src/cmd.c index 7a44ce2..d5f5c3d 100644 --- a/src/cmd.c +++ b/src/cmd.c @@ -35,6 +35,7 @@ #include #include "sanlock_internal.h" +#include "sanlock_admin.h" #include "sanlock_sock.h" #include "diskio.h" #include "log.h" @@ -844,6 +845,8 @@ static void cmd_examine(struct task *task GNUC_UNUSED, struct cmd_args *ca) static void cmd_add_lockspace(struct cmd_args *ca) { struct sanlk_lockspace lockspace; + struct space *sp; + int async = ca->header.cmd_flags & SANLK_ADD_ASYNC; int fd, rv, result; fd = client[ca->ci_in].fd; @@ -856,16 +859,31 @@ static void cmd_add_lockspace(struct cmd_args *ca) goto reply; } - log_debug("cmd_add_lockspace %d,%d %.48s:%llu:%s:%llu", + log_debug("cmd_add_lockspace %d,%d %.48s:%llu:%s:%llu flags %x", ca->ci_in, fd, lockspace.name, (unsigned long long)lockspace.host_id, lockspace.host_id_disk.path, - (unsigned long long)lockspace.host_id_disk.offset); + (unsigned long long)lockspace.host_id_disk.offset, + ca->header.cmd_flags); + + rv = add_lockspace_start(&lockspace, &sp); + if (rv < 0) { + result = rv; + goto reply; + } + + if (async) { + result = rv; + log_debug("cmd_add_lockspace %d,%d async done %d", ca->ci_in, fd, result); + send_result(fd, &ca->header, result); + client_resume(ca->ci_in); + add_lockspace_wait(sp); + return; + } - result = add_lockspace(&lockspace); + result = add_lockspace_wait(sp); reply: log_debug("cmd_add_lockspace %d,%d done %d", ca->ci_in, fd, result); - send_result(fd, &ca->header, result); client_resume(ca->ci_in); } @@ -947,7 +965,9 @@ static void cmd_inq_lockspace(struct cmd_args *ca) static void cmd_rem_lockspace(struct cmd_args *ca) { struct sanlk_lockspace lockspace; + int async = ca->header.cmd_flags & SANLK_REM_ASYNC; int fd, rv, result; + unsigned int space_id; fd = client[ca->ci_in].fd; @@ -959,13 +979,27 @@ static void cmd_rem_lockspace(struct cmd_args *ca) goto reply; } - log_debug("cmd_rem_lockspace %d,%d %.48s", - ca->ci_in, fd, lockspace.name); + log_debug("cmd_rem_lockspace %d,%d %.48s flags %x", + ca->ci_in, fd, lockspace.name, ca->header.cmd_flags); + + rv = rem_lockspace_start(&lockspace, &space_id); + if (rv < 0) { + result = rv; + goto reply; + } - result = rem_lockspace(&lockspace); + if (async) { + result = rv; + log_debug("cmd_rem_lockspace %d,%d async done %d", ca->ci_in, fd, result); + send_result(fd, &ca->header, result); + client_resume(ca->ci_in); + rem_lockspace_wait(&lockspace, space_id); + return; + } + + result = rem_lockspace_wait(&lockspace, space_id); reply: log_debug("cmd_rem_lockspace %d,%d done %d", ca->ci_in, fd, result); - send_result(fd, &ca->header, result); client_resume(ca->ci_in); } diff --git a/src/lockspace.c b/src/lockspace.c index 3432555..0e9ad87 100644 --- a/src/lockspace.c +++ b/src/lockspace.c @@ -574,16 +574,10 @@ static void free_sp(struct space *sp) free(sp); } -/* - * When this function returns, it needs to be safe to being processing lease - * requests and allowing pid's to run, so we need to own our host_id, and the - * watchdog needs to be active watching our host_id renewals. - */ - -int add_lockspace(struct sanlk_lockspace *ls) +int add_lockspace_start(struct sanlk_lockspace *ls, struct space **sp_out) { struct space *sp, *sp2; - int rv, result; + int rv; if (!ls->name[0] || !ls->host_id || !ls->host_id_disk.path[0]) { log_error("add_lockspace bad args id %llu name %zu path %zu", @@ -670,6 +664,22 @@ int add_lockspace(struct sanlk_lockspace *ls) goto fail_del; } + *sp_out = sp; + return 0; + + fail_del: + pthread_mutex_lock(&spaces_mutex); + list_del(&sp->list); + pthread_mutex_unlock(&spaces_mutex); + fail_free: + free_sp(sp); + return rv; +} + +int add_lockspace_wait(struct space *sp) +{ + int rv, result; + while (1) { pthread_mutex_lock(&sp->mutex); result = sp->lease_status.acquire_last_result; @@ -703,7 +713,6 @@ int add_lockspace(struct sanlk_lockspace *ls) pthread_mutex_lock(&spaces_mutex); list_del(&sp->list); pthread_mutex_unlock(&spaces_mutex); - fail_free: free_sp(sp); return rv; } @@ -736,11 +745,11 @@ int inq_lockspace(struct sanlk_lockspace *ls) return rv; } -int rem_lockspace(struct sanlk_lockspace *ls) +int rem_lockspace_start(struct sanlk_lockspace *ls, unsigned int *space_id) { - struct space *sp, *sp2; + struct space *sp; unsigned int id; - int rv, done; + int rv; pthread_mutex_lock(&spaces_mutex); @@ -769,15 +778,41 @@ int rem_lockspace(struct sanlk_lockspace *ls) goto out; } + /* + * Removal happens in a round about way: + * - we set external_remove + * - main_loop sees external_remove and sets space_dead, killing_pids + * - main_loop sees killing_pids and all pids dead, sets thread_stop, + * and moves sp from spaces to spaces_rem + * - main_loop calls free_lockspaces(0), which joins any + * lockspace_thread that is done, and then frees sp + * + * Once we release spaces_mutex, the sp could be freed any time, + * so we can't touch it. Use its space_id to check for completion. + */ + sp->external_remove = 1; id = sp->space_id; pthread_mutex_unlock(&spaces_mutex); + *space_id = id; + rv = 0; + out: + return rv; +} + +/* check for matching space_id in case the lockspace is added again */ + +int rem_lockspace_wait(struct sanlk_lockspace *ls, unsigned int space_id) +{ + struct space *sp; + int done; + while (1) { pthread_mutex_lock(&spaces_mutex); - sp2 = _search_space(ls->name, (struct sync_disk *)&ls->host_id_disk, ls->host_id, - &spaces, &spaces_rem, NULL); - if (sp2 && sp2->space_id == id) + sp = _search_space(ls->name, (struct sync_disk *)&ls->host_id_disk, ls->host_id, + &spaces, &spaces_rem, NULL); + if (sp && (sp->space_id == space_id)) done = 0; else done = 1; @@ -787,9 +822,7 @@ int rem_lockspace(struct sanlk_lockspace *ls) break; sleep(1); } - rv = 0; - out: - return rv; + return 0; } /* diff --git a/src/lockspace.h b/src/lockspace.h index af7d8c2..cf456ed 100644 --- a/src/lockspace.h +++ b/src/lockspace.h @@ -19,9 +19,11 @@ int test_id_bit(int host_id, char *bitmap); void set_id_bit(int host_id, char *bitmap, char *c); int check_our_lease(struct task *task, struct space *sp, int *check_all, char *check_buf); void check_other_leases(struct task *task, struct space *sp, char *buf); -int add_lockspace(struct sanlk_lockspace *ls); +int add_lockspace_start(struct sanlk_lockspace *ls, struct space **sp_out); +int add_lockspace_wait(struct space *sp); int inq_lockspace(struct sanlk_lockspace *ls); -int rem_lockspace(struct sanlk_lockspace *ls); +int rem_lockspace_start(struct sanlk_lockspace *ls, unsigned int *space_id); +int rem_lockspace_wait(struct sanlk_lockspace *ls, unsigned int space_id); void free_lockspaces(int wait); #endif diff --git a/src/sanlock_admin.h b/src/sanlock_admin.h index 2ec2795..64d654c 100644 --- a/src/sanlock_admin.h +++ b/src/sanlock_admin.h @@ -10,6 +10,12 @@ #ifndef __SANLOCK_ADMIN_H__ #define __SANLOCK_ADMIN_H__ +/* add flags */ +#define SANLK_ADD_ASYNC 0x00000001 + +/* rem flags */ +#define SANLK_REM_ASYNC 0x00000001 + /* * add_lockspace returns: * 0: the lockspace has been added successfully diff --git a/tests/sanlk_load.c b/tests/sanlk_load.c index cc511f2..8e87ab7 100644 --- a/tests/sanlk_load.c +++ b/tests/sanlk_load.c @@ -277,15 +277,20 @@ static int add_lockspace(int i) { struct sanlk_lockspace ls; int rv; + int async = !(i % 2); + uint32_t flags = 0; memset(&ls, 0, sizeof(ls)); sprintf(ls.host_id_disk.path, "%s%d", lock_disk_base, i); sprintf(ls.name, "lockspace%d", i); ls.host_id = our_hostid; + if (async) + flags = SANLK_ADD_ASYNC; + printf("add lockspace%d...\n", i); - rv = sanlock_add_lockspace(&ls, 0); + rv = sanlock_add_lockspace(&ls, flags); if (rv == -EEXIST) return 0; @@ -295,6 +300,24 @@ static int add_lockspace(int i) return -1; } + if (!async) + goto out; + + while (1) { + rv = sanlock_inq_lockspace(&ls, 0); + if (!rv) + goto out; + + if (rv == -EINPROGRESS) { + sleep(2); + continue; + } + + log_error("sanlock_inq_lockspace error %d", rv); + return -1; + } + + out: printf("add done\n"); return 0; }