mds_client.h 13 KB
Newer Older
Sage Weil's avatar
Sage Weil committed
1 2 3 4
#ifndef _FS_CEPH_MDS_CLIENT_H
#define _FS_CEPH_MDS_CLIENT_H

#include <linux/completion.h>
5
#include <linux/kref.h>
Sage Weil's avatar
Sage Weil committed
6 7
#include <linux/list.h>
#include <linux/mutex.h>
8
#include <linux/rbtree.h>
Sage Weil's avatar
Sage Weil committed
9 10
#include <linux/spinlock.h>

11 12 13
#include <linux/ceph/types.h>
#include <linux/ceph/messenger.h>
#include <linux/ceph/mdsmap.h>
14
#include <linux/ceph/auth.h>
Sage Weil's avatar
Sage Weil committed
15 16 17 18 19 20 21 22 23

/*
 * Some lock dependencies:
 *
 * session->s_mutex
 *         mdsc->mutex
 *
 *         mdsc->snap_rwsem
 *
24
 *         ci->i_ceph_lock
Sage Weil's avatar
Sage Weil committed
25 26 27 28 29
 *                 mdsc->snap_flush_lock
 *                 mdsc->cap_delay_lock
 *
 */

30
struct ceph_fs_client;
Sage Weil's avatar
Sage Weil committed
31 32 33 34 35 36 37 38
struct ceph_cap;

/*
 * parsed info about a single inode.  pointers are into the encoded
 * on-wire structures within the mds reply message payload.
 */
struct ceph_mds_reply_info_in {
	struct ceph_mds_reply_inode *in;
39
	struct ceph_dir_layout dir_layout;
Sage Weil's avatar
Sage Weil committed
40 41 42 43
	u32 symlink_len;
	char *symlink;
	u32 xattr_len;
	char *xattr_data;
44 45 46
	u64 inline_version;
	u32 inline_len;
	char *inline_data;
47
	u32 pool_ns_len;
Sage Weil's avatar
Sage Weil committed
48 49
};

50 51 52 53 54
struct ceph_mds_reply_dir_entry {
	char                          *name;
	u32                           name_len;
	struct ceph_mds_reply_lease   *lease;
	struct ceph_mds_reply_info_in inode;
55
	loff_t			      offset;
56 57
};

Sage Weil's avatar
Sage Weil committed
58
/*
59 60 61 62
 * parsed info about an mds reply, including information about
 * either: 1) the target inode and/or its parent directory and dentry,
 * and directory contents (for readdir results), or
 * 2) the file range lock info (for fcntl F_GETLK results).
Sage Weil's avatar
Sage Weil committed
63 64 65 66
 */
struct ceph_mds_reply_info_parsed {
	struct ceph_mds_reply_head    *head;

67
	/* trace */
Sage Weil's avatar
Sage Weil committed
68 69 70 71 72 73
	struct ceph_mds_reply_info_in diri, targeti;
	struct ceph_mds_reply_dirfrag *dirfrag;
	char                          *dname;
	u32                           dname_len;
	struct ceph_mds_reply_lease   *dlease;

74 75 76 77 78 79 80 81
	/* extra */
	union {
		/* for fcntl F_GETLK results */
		struct ceph_filelock *filelock_reply;

		/* for readdir results */
		struct {
			struct ceph_mds_reply_dirfrag *dir_dir;
82
			size_t			      dir_buf_size;
83
			int                           dir_nr;
84 85 86
			bool			      dir_complete;
			bool			      dir_end;
			bool			      hash_order;
87
			struct ceph_mds_reply_dir_entry  *dir_entries;
88
		};
89 90 91 92 93 94

		/* for create results */
		struct {
			bool has_create_ino;
			u64 ino;
		};
95
	};
Sage Weil's avatar
Sage Weil committed
96 97 98 99 100 101 102 103 104 105 106

	/* encoded blob describing snapshot contexts for certain
	   operations (e.g., open) */
	void *snapblob;
	int snapblob_len;
};


/*
 * cap releases are batched and sent to the MDS en masse.
 */
107
#define CEPH_CAPS_PER_RELEASE ((PAGE_SIZE -			\
Sage Weil's avatar
Sage Weil committed
108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134
				sizeof(struct ceph_mds_cap_release)) /	\
			       sizeof(struct ceph_mds_cap_item))


/*
 * state associated with each MDS<->client session
 */
enum {
	CEPH_MDS_SESSION_NEW = 1,
	CEPH_MDS_SESSION_OPENING = 2,
	CEPH_MDS_SESSION_OPEN = 3,
	CEPH_MDS_SESSION_HUNG = 4,
	CEPH_MDS_SESSION_CLOSING = 5,
	CEPH_MDS_SESSION_RESTARTING = 6,
	CEPH_MDS_SESSION_RECONNECTING = 7,
};

struct ceph_mds_session {
	struct ceph_mds_client *s_mdsc;
	int               s_mds;
	int               s_state;
	unsigned long     s_ttl;      /* time until mds kills us */
	u64               s_seq;      /* incoming msg seq # */
	struct mutex      s_mutex;    /* serialize session messages */

	struct ceph_connection s_con;

135
	struct ceph_auth_handshake s_auth;
136

137 138
	/* protected by s_gen_ttl_lock */
	spinlock_t        s_gen_ttl_lock;
Sage Weil's avatar
Sage Weil committed
139 140
	u32               s_cap_gen;  /* inc each time we get mds stale msg */
	unsigned long     s_cap_ttl;  /* when session caps expire */
141 142 143

	/* protected by s_cap_lock */
	spinlock_t        s_cap_lock;
Sage Weil's avatar
Sage Weil committed
144 145 146
	struct list_head  s_caps;     /* all caps issued by this session */
	int               s_nr_caps, s_trim_caps;
	int               s_num_cap_releases;
147
	int		  s_cap_reconnect;
148
	int		  s_readonly;
Sage Weil's avatar
Sage Weil committed
149
	struct list_head  s_cap_releases; /* waiting cap_release messages */
150
	struct ceph_cap  *s_cap_iterator;
Sage Weil's avatar
Sage Weil committed
151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179

	/* protected by mutex */
	struct list_head  s_cap_flushing;     /* inodes w/ flushing caps */
	struct list_head  s_cap_snaps_flushing;
	unsigned long     s_renew_requested; /* last time we sent a renew req */
	u64               s_renew_seq;

	atomic_t          s_ref;
	struct list_head  s_waiting;  /* waiting requests */
	struct list_head  s_unsafe;   /* unsafe requests */
};

/*
 * modes of choosing which MDS to send a request to
 */
enum {
	USE_ANY_MDS,
	USE_RANDOM_MDS,
	USE_AUTH_MDS,   /* prefer authoritative mds for this metadata item */
};

struct ceph_mds_request;
struct ceph_mds_client;

/*
 * request completion callback
 */
typedef void (*ceph_mds_request_callback_t) (struct ceph_mds_client *mdsc,
					     struct ceph_mds_request *req);
180 181 182 183 184
/*
 * wait for request completion callback
 */
typedef int (*ceph_mds_request_wait_callback_t) (struct ceph_mds_client *mdsc,
						 struct ceph_mds_request *req);
Sage Weil's avatar
Sage Weil committed
185 186 187 188 189 190

/*
 * an in-flight mds request
 */
struct ceph_mds_request {
	u64 r_tid;                   /* transaction id */
191
	struct rb_node r_node;
192
	struct ceph_mds_client *r_mdsc;
Sage Weil's avatar
Sage Weil committed
193 194 195 196 197 198 199

	int r_op;                    /* mds op code */

	/* operation on what? */
	struct inode *r_inode;              /* arg1 */
	struct dentry *r_dentry;            /* arg1 */
	struct dentry *r_old_dentry;        /* arg2: rename from or link from */
200
	struct inode *r_old_dentry_dir;     /* arg2: old dentry's parent dir */
Sage Weil's avatar
Sage Weil committed
201 202 203 204 205 206
	char *r_path1, *r_path2;
	struct ceph_vino r_ino1, r_ino2;

	struct inode *r_locked_dir; /* dir (if any) i_mutex locked by vfs */
	struct inode *r_target_inode;       /* resulting inode */

207 208
	struct mutex r_fill_mutex;

Sage Weil's avatar
Sage Weil committed
209 210
	union ceph_mds_request_args r_args;
	int r_fmode;        /* file mode, if expecting cap */
211 212
	kuid_t r_uid;
	kgid_t r_gid;
213
	struct timespec r_stamp;
Sage Weil's avatar
Sage Weil committed
214 215 216 217 218 219 220

	/* for choosing which mds to send this request to */
	int r_direct_mode;
	u32 r_direct_hash;      /* choose dir frag based on this dentry hash */
	bool r_direct_is_hash;  /* true if r_direct_hash is valid */

	/* data payload is used for xattr ops */
221
	struct ceph_pagelist *r_pagelist;
Sage Weil's avatar
Sage Weil committed
222 223 224 225 226 227 228 229 230

	/* what caps shall we drop? */
	int r_inode_drop, r_inode_unless;
	int r_dentry_drop, r_dentry_unless;
	int r_old_dentry_drop, r_old_dentry_unless;
	struct inode *r_old_inode;
	int r_old_inode_drop, r_old_inode_unless;

	struct ceph_msg  *r_request;  /* original request */
231
	int r_request_release_offset;
Sage Weil's avatar
Sage Weil committed
232 233
	struct ceph_msg  *r_reply;
	struct ceph_mds_reply_info_parsed r_reply_info;
234
	struct page *r_locked_page;
Sage Weil's avatar
Sage Weil committed
235
	int r_err;
236
	bool r_aborted;
Sage Weil's avatar
Sage Weil committed
237

238
	unsigned long r_timeout;  /* optional.  jiffies, 0 is "wait forever" */
Sage Weil's avatar
Sage Weil committed
239 240 241 242 243 244 245 246
	unsigned long r_started;  /* start time to measure timeout against */
	unsigned long r_request_started; /* start time for mds request only,
					    used to measure lease durations */

	/* link unsafe requests to parent directory, for fsync */
	struct inode	*r_unsafe_dir;
	struct list_head r_unsafe_dir_item;

247 248 249
	/* unsafe requests that modify the target inode */
	struct list_head r_unsafe_target_item;

Sage Weil's avatar
Sage Weil committed
250 251 252 253 254
	struct ceph_mds_session *r_session;

	int               r_attempts;   /* resend attempts */
	int               r_num_fwd;    /* number of forward attempts */
	int               r_resend_mds; /* mds to resend to next, if any*/
255
	u32               r_sent_on_mseq; /* cap mseq request was sent at*/
Sage Weil's avatar
Sage Weil committed
256

257
	struct kref       r_kref;
Sage Weil's avatar
Sage Weil committed
258 259 260 261
	struct list_head  r_wait;
	struct completion r_completion;
	struct completion r_safe_completion;
	ceph_mds_request_callback_t r_callback;
262
	ceph_mds_request_wait_callback_t r_wait_for_completion;
Sage Weil's avatar
Sage Weil committed
263
	struct list_head  r_unsafe_item;  /* per-session unsafe list item */
264
	bool		  r_got_unsafe, r_got_safe, r_got_result;
Sage Weil's avatar
Sage Weil committed
265 266

	bool              r_did_prepopulate;
Yan, Zheng's avatar
Yan, Zheng committed
267 268 269
	long long	  r_dir_release_cnt;
	long long	  r_dir_ordered_cnt;
	int		  r_readdir_cache_idx;
Sage Weil's avatar
Sage Weil committed
270 271 272 273 274 275
	u32               r_readdir_offset;

	struct ceph_cap_reservation r_caps_reservation;
	int r_num_caps;
};

276 277 278 279 280 281
struct ceph_pool_perm {
	struct rb_node node;
	u32 pool;
	int perm;
};

Sage Weil's avatar
Sage Weil committed
282 283 284 285
/*
 * mds client state
 */
struct ceph_mds_client {
286
	struct ceph_fs_client  *fsc;
Sage Weil's avatar
Sage Weil committed
287 288 289
	struct mutex            mutex;         /* all nested structures */

	struct ceph_mdsmap      *mdsmap;
290 291
	struct completion       safe_umount_waiters;
	wait_queue_head_t       session_close_wq;
Sage Weil's avatar
Sage Weil committed
292 293 294
	struct list_head        waiting_for_map;

	struct ceph_mds_session **sessions;    /* NULL for mds if no session */
295
	atomic_t		num_sessions;
Sage Weil's avatar
Sage Weil committed
296 297 298 299 300 301 302 303 304 305
	int                     max_sessions;  /* len of s_mds_sessions */
	int                     stopping;      /* true if shutting down */

	/*
	 * snap_rwsem will cover cap linkage into snaprealms, and
	 * realm snap contexts.  (later, we can do per-realm snap
	 * contexts locks..)  the empty list contains realms with no
	 * references (implying they contain no inodes with caps) that
	 * should be destroyed.
	 */
306
	u64			last_snap_seq;
Sage Weil's avatar
Sage Weil committed
307
	struct rw_semaphore     snap_rwsem;
Sage Weil's avatar
Sage Weil committed
308
	struct rb_root          snap_realms;
Sage Weil's avatar
Sage Weil committed
309 310 311 312
	struct list_head        snap_empty;
	spinlock_t              snap_empty_lock;  /* protect snap_empty */

	u64                    last_tid;      /* most recent mds request */
313 314
	u64                    oldest_tid;    /* oldest incomplete mds request,
						 excluding setfilelock requests */
315
	struct rb_root         request_tree;  /* pending mds requests */
Sage Weil's avatar
Sage Weil committed
316 317 318 319 320 321 322
	struct delayed_work    delayed_work;  /* delayed work */
	unsigned long    last_renew_caps;  /* last time we renewed our caps */
	struct list_head cap_delay_list;   /* caps with delayed release */
	spinlock_t       cap_delay_lock;   /* protects cap_delay_list */
	struct list_head snap_flush_list;  /* cap_snaps ready to flush */
	spinlock_t       snap_flush_lock;

323
	u64               last_cap_flush_tid;
324
	struct rb_root    cap_flush_tree;
Sage Weil's avatar
Sage Weil committed
325
	struct list_head  cap_dirty;        /* inodes with dirty caps */
326
	struct list_head  cap_dirty_migrating; /* ...that are migration... */
Sage Weil's avatar
Sage Weil committed
327 328 329 330
	int               num_cap_flushing; /* # caps we are flushing */
	spinlock_t        cap_dirty_lock;   /* protects above items */
	wait_queue_head_t cap_flushing_wq;

331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350
	/*
	 * Cap reservations
	 *
	 * Maintain a global pool of preallocated struct ceph_caps, referenced
	 * by struct ceph_caps_reservations.  This ensures that we preallocate
	 * memory needed to successfully process an MDS response.  (If an MDS
	 * sends us cap information and we fail to process it, we will have
	 * problems due to the client and MDS being out of sync.)
	 *
	 * Reservations are 'owned' by a ceph_cap_reservation context.
	 */
	spinlock_t	caps_list_lock;
	struct		list_head caps_list; /* unused (reserved or
						unreserved) */
	int		caps_total_count;    /* total caps allocated */
	int		caps_use_count;      /* in use */
	int		caps_reserve_count;  /* unused, reserved */
	int		caps_avail_count;    /* unused, unreserved */
	int		caps_min_count;      /* keep at least this many
						(unreserved) */
Sage Weil's avatar
Sage Weil committed
351 352 353
	spinlock_t	  dentry_lru_lock;
	struct list_head  dentry_lru;
	int		  num_dentry;
354 355 356

	struct rw_semaphore     pool_perm_rwsem;
	struct rb_root		pool_perm_tree;
Sage Weil's avatar
Sage Weil committed
357 358 359 360 361 362 363 364 365 366 367 368 369 370
};

extern const char *ceph_mds_op_name(int op);

extern struct ceph_mds_session *
__ceph_lookup_mds_session(struct ceph_mds_client *, int mds);

static inline struct ceph_mds_session *
ceph_get_mds_session(struct ceph_mds_session *s)
{
	atomic_inc(&s->s_ref);
	return s;
}

371 372
extern const char *ceph_session_state_name(int s);

Sage Weil's avatar
Sage Weil committed
373 374 375 376 377
extern void ceph_put_mds_session(struct ceph_mds_session *s);

extern int ceph_send_msg_mds(struct ceph_mds_client *mdsc,
			     struct ceph_msg *msg, int mds);

378
extern int ceph_mdsc_init(struct ceph_fs_client *fsc);
Sage Weil's avatar
Sage Weil committed
379
extern void ceph_mdsc_close_sessions(struct ceph_mds_client *mdsc);
380
extern void ceph_mdsc_force_umount(struct ceph_mds_client *mdsc);
381
extern void ceph_mdsc_destroy(struct ceph_fs_client *fsc);
Sage Weil's avatar
Sage Weil committed
382 383 384 385 386

extern void ceph_mdsc_sync(struct ceph_mds_client *mdsc);

extern void ceph_mdsc_lease_release(struct ceph_mds_client *mdsc,
				    struct inode *inode,
Sage Weil's avatar
Sage Weil committed
387
				    struct dentry *dn);
Sage Weil's avatar
Sage Weil committed
388

389
extern void ceph_invalidate_dir_request(struct ceph_mds_request *req);
390 391
extern int ceph_alloc_readdir_reply_buffer(struct ceph_mds_request *req,
					   struct inode *dir);
Sage Weil's avatar
Sage Weil committed
392 393 394 395 396 397 398 399 400
extern struct ceph_mds_request *
ceph_mdsc_create_request(struct ceph_mds_client *mdsc, int op, int mode);
extern void ceph_mdsc_submit_request(struct ceph_mds_client *mdsc,
				     struct ceph_mds_request *req);
extern int ceph_mdsc_do_request(struct ceph_mds_client *mdsc,
				struct inode *dir,
				struct ceph_mds_request *req);
static inline void ceph_mdsc_get_request(struct ceph_mds_request *req)
{
401 402 403 404 405 406
	kref_get(&req->r_kref);
}
extern void ceph_mdsc_release_request(struct kref *kref);
static inline void ceph_mdsc_put_request(struct ceph_mds_request *req)
{
	kref_put(&req->r_kref, ceph_mdsc_release_request);
Sage Weil's avatar
Sage Weil committed
407 408
}

409 410 411
extern void ceph_send_cap_releases(struct ceph_mds_client *mdsc,
				   struct ceph_mds_session *session);

Sage Weil's avatar
Sage Weil committed
412 413 414 415 416 417 418 419 420 421 422 423 424 425
extern void ceph_mdsc_pre_umount(struct ceph_mds_client *mdsc);

extern char *ceph_mdsc_build_path(struct dentry *dentry, int *plen, u64 *base,
				  int stop_on_nosnap);

extern void __ceph_mdsc_drop_dentry_lease(struct dentry *dentry);
extern void ceph_mdsc_lease_send_msg(struct ceph_mds_session *session,
				     struct inode *inode,
				     struct dentry *dentry, char action,
				     u32 seq);

extern void ceph_mdsc_handle_map(struct ceph_mds_client *mdsc,
				 struct ceph_msg *msg);

426 427
extern struct ceph_mds_session *
ceph_mdsc_open_export_target_session(struct ceph_mds_client *mdsc, int target);
428 429 430
extern void ceph_mdsc_open_export_target_sessions(struct ceph_mds_client *mdsc,
					  struct ceph_mds_session *session);

Sage Weil's avatar
Sage Weil committed
431
#endif