From ef550f6f4f6c9345a27ec85d98f4f7de1adce79c Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 25 Mar 2011 13:27:48 -0700 Subject: [PATCH 1/7] ceph: flush msgr_wq during mds_client shutdown The release method for mds connections uses a backpointer to the mds_client, so we need to flush the workqueue of any pending work (and ceph_connection references) prior to freeing the mds_client. This fixes an oops easily triggered under UML by while true ; do mount ... ; umount ... ; done Also fix an outdated comment: the flush in ceph_destroy_client only flushes OSD connections out. This bug is basically an artifact of the ceph -> ceph+libceph conversion. Signed-off-by: Sage Weil --- fs/ceph/mds_client.c | 6 ++++++ net/ceph/ceph_common.c | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c index a1ee8fa3a8e7..f60b07b0feb0 100644 --- a/fs/ceph/mds_client.c +++ b/fs/ceph/mds_client.c @@ -3215,9 +3215,15 @@ void ceph_mdsc_destroy(struct ceph_fs_client *fsc) { struct ceph_mds_client *mdsc = fsc->mdsc; + dout("mdsc_destroy %p\n", mdsc); ceph_mdsc_stop(mdsc); + + /* flush out any connection work with references to us */ + ceph_msgr_flush(); + fsc->mdsc = NULL; kfree(mdsc); + dout("mdsc_destroy %p done\n", mdsc); } diff --git a/net/ceph/ceph_common.c b/net/ceph/ceph_common.c index 95f96ab94bba..9bbb356b12e7 100644 --- a/net/ceph/ceph_common.c +++ b/net/ceph/ceph_common.c @@ -394,8 +394,8 @@ void ceph_destroy_client(struct ceph_client *client) ceph_osdc_stop(&client->osdc); /* - * make sure mds and osd connections close out before destroying - * the auth module, which is needed to free those connections' + * make sure osd connections close out before destroying the + * auth module, which is needed to free those connections' * ceph_authorizers. */ ceph_msgr_flush(); From 6b0ae4097c1ebad98c7b33f83b4ca7e93890ed12 Mon Sep 17 00:00:00 2001 From: Mariusz Kozlowski Date: Sat, 26 Mar 2011 19:29:34 +0100 Subject: [PATCH 2/7] ceph: fix possible NULL pointer dereference This patch fixes 'event_work' dereference before it is checked for NULL. Signed-off-by: Mariusz Kozlowski Signed-off-by: Sage Weil --- net/ceph/osd_client.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c index 02212ed50852..b6776cb627cc 100644 --- a/net/ceph/osd_client.c +++ b/net/ceph/osd_client.c @@ -1602,11 +1602,11 @@ void handle_watch_notify(struct ceph_osd_client *osdc, struct ceph_msg *msg) cookie, ver, event); if (event) { event_work = kmalloc(sizeof(*event_work), GFP_NOIO); - INIT_WORK(&event_work->work, do_event_work); if (!event_work) { dout("ERROR: could not allocate event_work\n"); goto done_err; } + INIT_WORK(&event_work->work, do_event_work); event_work->event = event; event_work->ver = ver; event_work->notify_id = notify_id; From 234af26ff123dfb2aa48772124721b1354c8e0a5 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Tue, 29 Mar 2011 06:25:59 +0300 Subject: [PATCH 3/7] ceph: unlock on error in ceph_osdc_start_request() There was a missing unlock on the error path if __map_request() failed. Signed-off-by: Dan Carpenter Signed-off-by: Sage Weil --- net/ceph/osd_client.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c index b6776cb627cc..03740e8fc9d1 100644 --- a/net/ceph/osd_client.c +++ b/net/ceph/osd_client.c @@ -1672,7 +1672,7 @@ int ceph_osdc_start_request(struct ceph_osd_client *osdc, if (req->r_sent == 0) { rc = __map_request(osdc, req); if (rc < 0) - return rc; + goto out_unlock; if (req->r_osd == NULL) { dout("send_request %p no up osds in pg\n", req); ceph_monc_request_next_osdmap(&osdc->client->monc); @@ -1689,6 +1689,8 @@ int ceph_osdc_start_request(struct ceph_osd_client *osdc, } } } + +out_unlock: mutex_unlock(&osdc->request_mutex); up_read(&osdc->map_sem); return rc; From fbdb9190482fd83a3eb20cdeb0da454759f479d7 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 29 Mar 2011 12:11:06 -0700 Subject: [PATCH 4/7] libceph: fix null dereference when unregistering linger requests We should only clear r_osd if we are neither registered as a linger or a regular request. We may unregister as a linger while still registered as a regular request (e.g., in reset_osd). Incorrectly clearing r_osd there leads to a null pointer dereference in __send_request. Also simplify the parallel check in __unregister_request() where we just removed r_osd_item and know it's empty. Signed-off-by: Sage Weil --- net/ceph/osd_client.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c index 03740e8fc9d1..3b91d651fe08 100644 --- a/net/ceph/osd_client.c +++ b/net/ceph/osd_client.c @@ -837,8 +837,7 @@ static void __unregister_request(struct ceph_osd_client *osdc, dout("moving osd to %p lru\n", req->r_osd); __move_osd_to_lru(osdc, req->r_osd); } - if (list_empty(&req->r_osd_item) && - list_empty(&req->r_linger_item)) + if (list_empty(&req->r_linger_item)) req->r_osd = NULL; } @@ -883,7 +882,8 @@ static void __unregister_linger_request(struct ceph_osd_client *osdc, dout("moving osd to %p lru\n", req->r_osd); __move_osd_to_lru(osdc, req->r_osd); } - req->r_osd = NULL; + if (list_empty(&req->r_osd_item)) + req->r_osd = NULL; } } From 8323c3aa74cd92465350294567142d12ffdcc963 Mon Sep 17 00:00:00 2001 From: Tommi Virtanen Date: Fri, 25 Mar 2011 16:32:57 -0700 Subject: [PATCH 5/7] ceph: Move secret key parsing earlier. This makes the base64 logic be contained in mount option parsing, and prepares us for replacing the homebew key management with the kernel key retention service. Signed-off-by: Tommi Virtanen Signed-off-by: Sage Weil --- fs/ceph/super.c | 2 +- include/linux/ceph/auth.h | 4 ++-- include/linux/ceph/libceph.h | 2 +- net/ceph/auth.c | 8 +++---- net/ceph/auth_x.c | 8 ++++--- net/ceph/ceph_common.c | 43 ++++++++++++++++++++++++++++++------ net/ceph/crypto.c | 11 +++++++++ net/ceph/crypto.h | 2 ++ net/ceph/mon_client.c | 2 +- 9 files changed, 63 insertions(+), 19 deletions(-) diff --git a/fs/ceph/super.c b/fs/ceph/super.c index a9e78b4a258c..f2f77fd3c14c 100644 --- a/fs/ceph/super.c +++ b/fs/ceph/super.c @@ -353,7 +353,7 @@ static int ceph_show_options(struct seq_file *m, struct vfsmount *mnt) if (opt->name) seq_printf(m, ",name=%s", opt->name); - if (opt->secret) + if (opt->key) seq_puts(m, ",secret="); if (opt->mount_timeout != CEPH_MOUNT_TIMEOUT_DEFAULT) diff --git a/include/linux/ceph/auth.h b/include/linux/ceph/auth.h index 7fff521d7eb5..aa13392a7efb 100644 --- a/include/linux/ceph/auth.h +++ b/include/linux/ceph/auth.h @@ -67,12 +67,12 @@ struct ceph_auth_client { bool negotiating; /* true if negotiating protocol */ const char *name; /* entity name */ u64 global_id; /* our unique id in system */ - const char *secret; /* our secret key */ + const struct ceph_crypto_key *key; /* our secret key */ unsigned want_keys; /* which services we want */ }; extern struct ceph_auth_client *ceph_auth_init(const char *name, - const char *secret); + const struct ceph_crypto_key *key); extern void ceph_auth_destroy(struct ceph_auth_client *ac); extern void ceph_auth_reset(struct ceph_auth_client *ac); diff --git a/include/linux/ceph/libceph.h b/include/linux/ceph/libceph.h index 0d2e0fffb470..6365f041745b 100644 --- a/include/linux/ceph/libceph.h +++ b/include/linux/ceph/libceph.h @@ -61,7 +61,7 @@ struct ceph_options { pointer type of args */ int num_mon; char *name; - char *secret; + struct ceph_crypto_key *key; }; /* diff --git a/net/ceph/auth.c b/net/ceph/auth.c index 549c1f43e1d5..b4bf4ac090f1 100644 --- a/net/ceph/auth.c +++ b/net/ceph/auth.c @@ -35,12 +35,12 @@ static int ceph_auth_init_protocol(struct ceph_auth_client *ac, int protocol) /* * setup, teardown. */ -struct ceph_auth_client *ceph_auth_init(const char *name, const char *secret) +struct ceph_auth_client *ceph_auth_init(const char *name, const struct ceph_crypto_key *key) { struct ceph_auth_client *ac; int ret; - dout("auth_init name '%s' secret '%s'\n", name, secret); + dout("auth_init name '%s'\n", name); ret = -ENOMEM; ac = kzalloc(sizeof(*ac), GFP_NOFS); @@ -52,8 +52,8 @@ struct ceph_auth_client *ceph_auth_init(const char *name, const char *secret) ac->name = name; else ac->name = CEPH_AUTH_NAME_DEFAULT; - dout("auth_init name %s secret %s\n", ac->name, secret); - ac->secret = secret; + dout("auth_init name %s\n", ac->name); + ac->key = key; return ac; out: diff --git a/net/ceph/auth_x.c b/net/ceph/auth_x.c index 7fd5dfcf6e18..1587dc6010c6 100644 --- a/net/ceph/auth_x.c +++ b/net/ceph/auth_x.c @@ -662,14 +662,16 @@ int ceph_x_init(struct ceph_auth_client *ac) goto out; ret = -EINVAL; - if (!ac->secret) { + if (!ac->key) { pr_err("no secret set (for auth_x protocol)\n"); goto out_nomem; } - ret = ceph_crypto_key_unarmor(&xi->secret, ac->secret); - if (ret) + ret = ceph_crypto_key_clone(&xi->secret, ac->key); + if (ret < 0) { + pr_err("cannot clone key: %d\n", ret); goto out_nomem; + } xi->starting = true; xi->ticket_handlers = RB_ROOT; diff --git a/net/ceph/ceph_common.c b/net/ceph/ceph_common.c index 9bbb356b12e7..02e084f29d24 100644 --- a/net/ceph/ceph_common.c +++ b/net/ceph/ceph_common.c @@ -20,6 +20,7 @@ #include #include #include +#include "crypto.h" @@ -117,9 +118,29 @@ int ceph_compare_options(struct ceph_options *new_opt, if (ret) return ret; - ret = strcmp_null(opt1->secret, opt2->secret); - if (ret) - return ret; + if (opt1->key && !opt2->key) + return -1; + if (!opt1->key && opt2->key) + return 1; + if (opt1->key && opt2->key) { + if (opt1->key->type != opt2->key->type) + return -1; + if (opt1->key->created.tv_sec != opt2->key->created.tv_sec) + return -1; + if (opt1->key->created.tv_nsec != opt2->key->created.tv_nsec) + return -1; + if (opt1->key->len != opt2->key->len) + return -1; + if (opt1->key->key && !opt2->key->key) + return -1; + if (!opt1->key->key && opt2->key->key) + return 1; + if (opt1->key->key && opt2->key->key) { + ret = memcmp(opt1->key->key, opt2->key->key, opt1->key->len); + if (ret) + return ret; + } + } /* any matching mon ip implies a match */ for (i = 0; i < opt1->num_mon; i++) { @@ -203,7 +224,10 @@ void ceph_destroy_options(struct ceph_options *opt) { dout("destroy_options %p\n", opt); kfree(opt->name); - kfree(opt->secret); + if (opt->key) { + ceph_crypto_key_destroy(opt->key); + kfree(opt->key); + } kfree(opt); } EXPORT_SYMBOL(ceph_destroy_options); @@ -295,9 +319,14 @@ int ceph_parse_options(struct ceph_options **popt, char *options, GFP_KERNEL); break; case Opt_secret: - opt->secret = kstrndup(argstr[0].from, - argstr[0].to-argstr[0].from, - GFP_KERNEL); + opt->key = kzalloc(sizeof(*opt->key), GFP_KERNEL); + if (!opt->key) { + err = -ENOMEM; + goto out; + } + err = ceph_crypto_key_unarmor(opt->key, argstr[0].from); + if (err < 0) + goto out; break; /* misc */ diff --git a/net/ceph/crypto.c b/net/ceph/crypto.c index 7b505b0c983f..75f0893fa11f 100644 --- a/net/ceph/crypto.c +++ b/net/ceph/crypto.c @@ -9,6 +9,17 @@ #include #include "crypto.h" +int ceph_crypto_key_clone(struct ceph_crypto_key *dst, + const struct ceph_crypto_key *src) +{ + memcpy(dst, src, sizeof(struct ceph_crypto_key)); + dst->key = kmalloc(src->len, GFP_NOFS); + if (!dst->key) + return -ENOMEM; + memcpy(dst->key, src->key, src->len); + return 0; +} + int ceph_crypto_key_encode(struct ceph_crypto_key *key, void **p, void *end) { if (*p + sizeof(u16) + sizeof(key->created) + diff --git a/net/ceph/crypto.h b/net/ceph/crypto.h index f9eccace592b..6cf6edc91ec4 100644 --- a/net/ceph/crypto.h +++ b/net/ceph/crypto.h @@ -19,6 +19,8 @@ static inline void ceph_crypto_key_destroy(struct ceph_crypto_key *key) kfree(key->key); } +extern int ceph_crypto_key_clone(struct ceph_crypto_key *dst, + const struct ceph_crypto_key *src); extern int ceph_crypto_key_encode(struct ceph_crypto_key *key, void **p, void *end); extern int ceph_crypto_key_decode(struct ceph_crypto_key *key, diff --git a/net/ceph/mon_client.c b/net/ceph/mon_client.c index 8a079399174a..cbe31fa45508 100644 --- a/net/ceph/mon_client.c +++ b/net/ceph/mon_client.c @@ -759,7 +759,7 @@ int ceph_monc_init(struct ceph_mon_client *monc, struct ceph_client *cl) /* authentication */ monc->auth = ceph_auth_init(cl->options->name, - cl->options->secret); + cl->options->key); if (IS_ERR(monc->auth)) return PTR_ERR(monc->auth); monc->auth->want_keys = From e2c3d29b4295c3eec18294bc34f0c99a7b9ae413 Mon Sep 17 00:00:00 2001 From: Tommi Virtanen Date: Fri, 25 Mar 2011 16:40:48 -0700 Subject: [PATCH 6/7] libceph: Get secret from the kernel keys api when mounting with key=NAME. Signed-off-by: Tommi Virtanen Signed-off-by: Sage Weil --- net/ceph/Kconfig | 1 + net/ceph/ceph_common.c | 58 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/net/ceph/Kconfig b/net/ceph/Kconfig index ad424049b0cf..be683f2d401f 100644 --- a/net/ceph/Kconfig +++ b/net/ceph/Kconfig @@ -4,6 +4,7 @@ config CEPH_LIB select LIBCRC32C select CRYPTO_AES select CRYPTO + select KEYS default n help Choose Y or M here to include cephlib, which provides the diff --git a/net/ceph/ceph_common.c b/net/ceph/ceph_common.c index 02e084f29d24..c92bc8d50597 100644 --- a/net/ceph/ceph_common.c +++ b/net/ceph/ceph_common.c @@ -5,6 +5,8 @@ #include #include #include +#include +#include #include #include #include @@ -197,6 +199,7 @@ enum { Opt_fsid, Opt_name, Opt_secret, + Opt_key, Opt_ip, Opt_last_string, /* string args above */ @@ -213,6 +216,7 @@ static match_table_t opt_tokens = { {Opt_fsid, "fsid=%s"}, {Opt_name, "name=%s"}, {Opt_secret, "secret=%s"}, + {Opt_key, "key=%s"}, {Opt_ip, "ip=%s"}, /* string args above */ {Opt_noshare, "noshare"}, @@ -232,6 +236,50 @@ void ceph_destroy_options(struct ceph_options *opt) } EXPORT_SYMBOL(ceph_destroy_options); +/* get secret from key store */ +static int get_secret(struct ceph_crypto_key *dst, const char *name) { + struct key *ukey; + int key_err; + int err = 0; + struct user_key_payload *payload; + void *p; + + ukey = request_key(&key_type_user, name, NULL); + if (!ukey || IS_ERR(ukey)) { + /* request_key errors don't map nicely to mount(2) + errors; don't even try, but still printk */ + key_err = PTR_ERR(ukey); + switch (key_err) { + case -ENOKEY: + pr_warning("ceph: Mount failed due to key not found: %s\n", name); + break; + case -EKEYEXPIRED: + pr_warning("ceph: Mount failed due to expired key: %s\n", name); + break; + case -EKEYREVOKED: + pr_warning("ceph: Mount failed due to revoked key: %s\n", name); + break; + default: + pr_warning("ceph: Mount failed due to unknown key error" + " %d: %s\n", key_err, name); + } + err = -EPERM; + goto out; + } + + payload = ukey->payload.data; + p = payload->data; + err = ceph_crypto_key_decode(dst, &p, p + payload->datalen); + if (err) + goto out_key; + /* pass through, err is 0 */ + +out_key: + key_put(ukey); +out: + return err; +} + int ceph_parse_options(struct ceph_options **popt, char *options, const char *dev_name, const char *dev_name_end, int (*parse_extra_token)(char *c, void *private), @@ -328,6 +376,16 @@ int ceph_parse_options(struct ceph_options **popt, char *options, if (err < 0) goto out; break; + case Opt_key: + opt->key = kzalloc(sizeof(*opt->key), GFP_KERNEL); + if (!opt->key) { + err = -ENOMEM; + goto out; + } + err = get_secret(opt->key, argstr[0].from); + if (err < 0) + goto out; + break; /* misc */ case Opt_osdtimeout: From 4b2a58abd1e17c0ee53c8dded879e015917cca67 Mon Sep 17 00:00:00 2001 From: Tommi Virtanen Date: Mon, 28 Mar 2011 14:59:38 -0700 Subject: [PATCH 7/7] libceph: Create a new key type "ceph". This allows us to use existence of the key type as a feature test, from userspace. Signed-off-by: Tommi Virtanen Signed-off-by: Sage Weil --- include/keys/ceph-type.h | 8 ++++++ net/ceph/ceph_common.c | 21 ++++++++------ net/ceph/crypto.c | 62 ++++++++++++++++++++++++++++++++++++++++ net/ceph/crypto.h | 2 ++ 4 files changed, 85 insertions(+), 8 deletions(-) create mode 100644 include/keys/ceph-type.h diff --git a/include/keys/ceph-type.h b/include/keys/ceph-type.h new file mode 100644 index 000000000000..f69c4ac197a0 --- /dev/null +++ b/include/keys/ceph-type.h @@ -0,0 +1,8 @@ +#ifndef _KEYS_CEPH_TYPE_H +#define _KEYS_CEPH_TYPE_H + +#include + +extern struct key_type key_type_ceph; + +#endif diff --git a/net/ceph/ceph_common.c b/net/ceph/ceph_common.c index c92bc8d50597..132963abc266 100644 --- a/net/ceph/ceph_common.c +++ b/net/ceph/ceph_common.c @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include #include #include @@ -241,10 +241,9 @@ static int get_secret(struct ceph_crypto_key *dst, const char *name) { struct key *ukey; int key_err; int err = 0; - struct user_key_payload *payload; - void *p; + struct ceph_crypto_key *ckey; - ukey = request_key(&key_type_user, name, NULL); + ukey = request_key(&key_type_ceph, name, NULL); if (!ukey || IS_ERR(ukey)) { /* request_key errors don't map nicely to mount(2) errors; don't even try, but still printk */ @@ -267,9 +266,8 @@ static int get_secret(struct ceph_crypto_key *dst, const char *name) { goto out; } - payload = ukey->payload.data; - p = payload->data; - err = ceph_crypto_key_decode(dst, &p, p + payload->datalen); + ckey = ukey->payload.data; + err = ceph_crypto_key_clone(dst, ckey); if (err) goto out_key; /* pass through, err is 0 */ @@ -583,10 +581,14 @@ static int __init init_ceph_lib(void) if (ret < 0) goto out; - ret = ceph_msgr_init(); + ret = ceph_crypto_init(); if (ret < 0) goto out_debugfs; + ret = ceph_msgr_init(); + if (ret < 0) + goto out_crypto; + pr_info("loaded (mon/osd proto %d/%d, osdmap %d/%d %d/%d)\n", CEPH_MONC_PROTOCOL, CEPH_OSDC_PROTOCOL, CEPH_OSDMAP_VERSION, CEPH_OSDMAP_VERSION_EXT, @@ -594,6 +596,8 @@ static int __init init_ceph_lib(void) return 0; +out_crypto: + ceph_crypto_shutdown(); out_debugfs: ceph_debugfs_cleanup(); out: @@ -604,6 +608,7 @@ static void __exit exit_ceph_lib(void) { dout("exit_ceph_lib\n"); ceph_msgr_exit(); + ceph_crypto_shutdown(); ceph_debugfs_cleanup(); } diff --git a/net/ceph/crypto.c b/net/ceph/crypto.c index 75f0893fa11f..5a8009c9e0cd 100644 --- a/net/ceph/crypto.c +++ b/net/ceph/crypto.c @@ -5,7 +5,9 @@ #include #include #include +#include +#include #include #include "crypto.h" @@ -421,3 +423,63 @@ int ceph_encrypt2(struct ceph_crypto_key *secret, void *dst, size_t *dst_len, return -EINVAL; } } + +int ceph_key_instantiate(struct key *key, const void *data, size_t datalen) +{ + struct ceph_crypto_key *ckey; + int ret; + void *p; + + ret = -EINVAL; + if (datalen <= 0 || datalen > 32767 || !data) + goto err; + + ret = key_payload_reserve(key, datalen); + if (ret < 0) + goto err; + + ret = -ENOMEM; + ckey = kmalloc(sizeof(*ckey), GFP_KERNEL); + if (!ckey) + goto err; + + /* TODO ceph_crypto_key_decode should really take const input */ + p = (void*)data; + ret = ceph_crypto_key_decode(ckey, &p, (char*)data+datalen); + if (ret < 0) + goto err_ckey; + + key->payload.data = ckey; + return 0; + +err_ckey: + kfree(ckey); +err: + return ret; +} + +int ceph_key_match(const struct key *key, const void *description) +{ + return strcmp(key->description, description) == 0; +} + +void ceph_key_destroy(struct key *key) { + struct ceph_crypto_key *ckey = key->payload.data; + + ceph_crypto_key_destroy(ckey); +} + +struct key_type key_type_ceph = { + .name = "ceph", + .instantiate = ceph_key_instantiate, + .match = ceph_key_match, + .destroy = ceph_key_destroy, +}; + +int ceph_crypto_init(void) { + return register_key_type(&key_type_ceph); +} + +void ceph_crypto_shutdown(void) { + unregister_key_type(&key_type_ceph); +} diff --git a/net/ceph/crypto.h b/net/ceph/crypto.h index 6cf6edc91ec4..1919d1550d75 100644 --- a/net/ceph/crypto.h +++ b/net/ceph/crypto.h @@ -42,6 +42,8 @@ extern int ceph_encrypt2(struct ceph_crypto_key *secret, void *dst, size_t *dst_len, const void *src1, size_t src1_len, const void *src2, size_t src2_len); +extern int ceph_crypto_init(void); +extern void ceph_crypto_shutdown(void); /* armor.c */ extern int ceph_armor(char *dst, const char *src, const char *end);