-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathGenericObjectMap.h
More file actions
431 lines (354 loc) · 12.8 KB
/
Copy pathGenericObjectMap.h
File metadata and controls
431 lines (354 loc) · 12.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
// vim: ts=8 sw=2 smarttab
/*
* Ceph - scalable distributed file system
*
* Copyright (C) 2013 UnitedStack <[email protected]>
*
* Author: Haomai Wang <[email protected]>
*
* This is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License version 2.1, as published by the Free Software
* Foundation. See file COPYING.
*
*/
#ifndef CEPH_GENERICOBJECTMAP_H
#define CEPH_GENERICOBJECTMAP_H
#include "include/buffer.h"
#include <set>
#include <map>
#include <string>
#include <vector>
#include <boost/scoped_ptr.hpp>
#include "include/memory.h"
#include "ObjectMap.h"
#include "KeyValueDB.h"
#include "osd/osd_types.h"
#include "common/Mutex.h"
#include "common/Cond.h"
#include "common/simple_cache.hpp"
/**
* Genericobjectmap: Provide with key/value associated to ghobject_t APIs to caller
* and avoid concerning too much. Wrap and combine KeyValueDB/ObjectMap APIs
* with ghobject_t and adding clone capacity.
*
* Prefix space structure:
*
* - GHOBJECT_TO_SEQ: Contains leaf mapping from ghobject_t->Header(including
* hobj.seq and related metadata)
* - INTERN_PREFIX: GLOBAL_STATE_KEY - contains the global state
* @see State
* @see write_state
* @see init
* @see generate_new_header
* - INTERN_PREFIX + header_key(header->seq) + COMPLETE_PREFIX: see below
* - INTERN_PREFIX + header_key(header->seq) + PARENT_KEY
* : used to store parent header(same as headers in GHOBJECT_TO_SEQ)
* - USER_PREFIX + header_key(header->seq) + [CUSTOM_PREFIX]
* : key->value which set by callers
*
* For each node (represented by a header), we
* store three mappings: the key mapping, the complete mapping, and the parent.
* The complete mapping (COMPLETE_PREFIX space) is key->key. Each x->y entry in
* this mapping indicates that the key mapping contains all entries on [x,y).
* Note, max string is represented by "", so ""->"" indicates that the parent
* is unnecessary (@see rm_keys). When looking up a key not contained in the
* the complete set, we have to check the parent if we don't find it in the
* key set. During rm_keys, we copy keys from the parent and update the
* complete set to reflect the change @see rm_keys.
*/
// This class only provide basic read capacity, suggest inherit it to
// implement write transaction to use it. @see StripObjectMap
class GenericObjectMap {
public:
boost::scoped_ptr<KeyValueDB> db;
/**
* Serializes access to next_seq as well as the in_use set
*/
Mutex header_lock;
GenericObjectMap(KeyValueDB *db) : db(db), header_lock("GenericObjectMap") {}
int get(
const coll_t &cid,
const ghobject_t &oid,
const string &prefix,
map<string, bufferlist> *out
);
int get_keys(
const coll_t &cid,
const ghobject_t &oid,
const string &prefix,
set<string> *keys
);
int get_values(
const coll_t &cid,
const ghobject_t &oid,
const string &prefix,
const set<string> &keys,
map<string, bufferlist> *out
);
int check_keys(
const coll_t &cid,
const ghobject_t &oid,
const string &prefix,
const set<string> &keys,
set<string> *out
);
/// Read initial state from backing store
int init(bool upgrade = false);
/// Upgrade store to current version
int upgrade() {return 0;}
/// Consistency check, debug, there must be no parallel writes
bool check(std::ostream &out);
/// Util, list all objects, there must be no other concurrent access
int list_objects(const coll_t &cid, ghobject_t start, int max,
vector<ghobject_t> *objs, ///< [out] objects
ghobject_t *next);
ObjectMap::ObjectMapIterator get_iterator(const coll_t &cid,
const ghobject_t &oid,
const string &prefix);
KeyValueDB::Transaction get_transaction() { return db->get_transaction(); }
int submit_transaction(KeyValueDB::Transaction t) {
return db->submit_transaction(t);
}
int submit_transaction_sync(KeyValueDB::Transaction t) {
return db->submit_transaction_sync(t);
}
/// persistent state for store @see generate_header
struct State {
__u8 v;
uint64_t seq;
State() : v(0), seq(1) {}
State(uint64_t seq) : v(0), seq(seq) {}
void encode(bufferlist &bl) const {
ENCODE_START(1, 1, bl);
::encode(v, bl);
::encode(seq, bl);
ENCODE_FINISH(bl);
}
void decode(bufferlist::iterator &bl) {
DECODE_START(1, bl);
::decode(v, bl);
::decode(seq, bl);
DECODE_FINISH(bl);
}
void dump(Formatter *f) const {
f->dump_unsigned("seq", seq);
}
static void generate_test_instances(list<State*> &o) {
o.push_back(new State(0));
o.push_back(new State(20));
}
} state;
struct _Header {
uint64_t seq;
uint64_t parent;
uint64_t num_children;
coll_t cid;
ghobject_t oid;
// Used by successor
bufferlist data;
void encode(bufferlist &bl) const {
ENCODE_START(1, 1, bl);
::encode(seq, bl);
::encode(parent, bl);
::encode(num_children, bl);
::encode(cid, bl);
::encode(oid, bl);
::encode(data, bl);
ENCODE_FINISH(bl);
}
void decode(bufferlist::iterator &bl) {
DECODE_START(1, bl);
::decode(seq, bl);
::decode(parent, bl);
::decode(num_children, bl);
::decode(cid, bl);
::decode(oid, bl);
::decode(data, bl);
DECODE_FINISH(bl);
}
void dump(Formatter *f) const {
f->dump_unsigned("seq", seq);
f->dump_unsigned("parent", parent);
f->dump_unsigned("num_children", num_children);
f->dump_stream("coll") << cid;
f->dump_stream("oid") << oid;
}
_Header() : seq(0), parent(0), num_children(1) {}
};
typedef ceph::shared_ptr<_Header> Header;
Header lookup_header(const coll_t &cid, const ghobject_t &oid) {
Mutex::Locker l(header_lock);
return _lookup_header(cid, oid);
}
/// Lookup or create header for c oid
Header lookup_create_header(const coll_t &cid, const ghobject_t &oid,
KeyValueDB::Transaction t);
/// Set leaf node for c and oid to the value of header
void set_header(const coll_t &cid, const ghobject_t &oid, _Header &header,
KeyValueDB::Transaction t);
// Move all modify member function to "protect", in order to indicate these
// should be made use of by sub-class
void set_keys(
const Header header,
const string &prefix,
const map<string, bufferlist> &set,
KeyValueDB::Transaction t
);
int clear(
const Header header,
KeyValueDB::Transaction t
);
int rm_keys(
const Header header,
const string &prefix,
const set<string> &to_clear,
KeyValueDB::Transaction t
);
void clone(
const Header origin_header,
const coll_t &cid,
const ghobject_t &target,
KeyValueDB::Transaction t,
Header *old_header,
Header *new_header
);
void rename(
const Header header,
const coll_t &cid,
const ghobject_t &target,
KeyValueDB::Transaction t
);
/// Ensure that all previous operations are durable
int sync(const Header header, KeyValueDB::Transaction t);
static const string GLOBAL_STATE_KEY;
static const string PARENT_KEY;
static const string USER_PREFIX;
static const string INTERN_PREFIX;
static const string PARENT_PREFIX;
static const string COMPLETE_PREFIX;
static const string GHOBJECT_TO_SEQ_PREFIX;
static const string GHOBJECT_KEY_SEP_S;
static const char GHOBJECT_KEY_SEP_C;
static const char GHOBJECT_KEY_ENDING;
private:
/// Implicit lock on Header->seq
static string header_key(const coll_t &cid);
static string header_key(const coll_t &cid, const ghobject_t &oid);
static bool parse_header_key(const string &in, coll_t *c, ghobject_t *oid);
string seq_key(uint64_t seq) {
char buf[100];
snprintf(buf, sizeof(buf), "%.*" PRId64, (int)(2*sizeof(seq)), seq);
return string(buf);
}
string user_prefix(Header header, const string &prefix);
string complete_prefix(Header header);
string parent_seq_prefix(uint64_t seq);
class EmptyIteratorImpl : public ObjectMap::ObjectMapIteratorImpl {
public:
int seek_to_first() { return 0; }
int seek_to_last() { return 0; }
int upper_bound(const string &after) { return 0; }
int lower_bound(const string &to) { return 0; }
bool valid() { return false; }
int next() { assert(0); return 0; }
string key() { assert(0); return ""; }
bufferlist value() { assert(0); return bufferlist(); }
int status() { return 0; }
};
/// Iterator
class GenericObjectMapIteratorImpl : public ObjectMap::ObjectMapIteratorImpl {
public:
GenericObjectMap *map;
/// NOTE: implicit lock on header->seq AND for all ancestors
Header header;
/// parent_iter == NULL iff no parent
ceph::shared_ptr<GenericObjectMapIteratorImpl> parent_iter;
KeyValueDB::Iterator key_iter;
KeyValueDB::Iterator complete_iter;
/// cur_iter points to currently valid iterator
ceph::shared_ptr<ObjectMap::ObjectMapIteratorImpl> cur_iter;
int r;
/// init() called, key_iter, complete_iter, parent_iter filled in
bool ready;
/// past end
bool invalid;
string prefix;
GenericObjectMapIteratorImpl(GenericObjectMap *map, Header header,
const string &_prefix) : map(map), header(header), r(0), ready(false),
invalid(true), prefix(_prefix) { }
int seek_to_first();
int seek_to_last();
int upper_bound(const string &after);
int lower_bound(const string &to);
bool valid();
int next();
string key();
bufferlist value();
int status();
bool on_parent() {
return cur_iter == parent_iter;
}
/// skips to next valid parent entry
int next_parent();
/// Tests whether to_test is in complete region
int in_complete_region(const string &to_test, ///[in] key to test
string *begin, ///[out] beginning of region
string *end ///[out] end of region
); ///< @returns true if to_test is in the complete region, else false
private:
int init();
bool valid_parent();
int adjust();
};
protected:
typedef ceph::shared_ptr<GenericObjectMapIteratorImpl> GenericObjectMapIterator;
GenericObjectMapIterator _get_iterator(Header header, string prefix) {
return GenericObjectMapIterator(new GenericObjectMapIteratorImpl(this, header, prefix));
}
Header generate_new_header(const coll_t &cid, const ghobject_t &oid,
Header parent, KeyValueDB::Transaction t) {
Mutex::Locker l(header_lock);
return _generate_new_header(cid, oid, parent, t);
}
// Scan keys in header into out_keys and out_values (if nonnull)
int scan(Header header, const string &prefix, const set<string> &in_keys,
set<string> *out_keys, map<string, bufferlist> *out_values);
private:
/// Removes node corresponding to header
void clear_header(Header header, KeyValueDB::Transaction t);
/// Set node containing input to new contents
void set_parent_header(Header input, KeyValueDB::Transaction t);
/// Remove leaf node corresponding to oid in c
void remove_header(const coll_t &cid, const ghobject_t &oid, Header header,
KeyValueDB::Transaction t);
/**
* Generate new header for c oid with new seq number
*
* Has the side effect of syncronously saving the new GenericObjectMap state
*/
Header _generate_new_header(const coll_t &cid, const ghobject_t &oid,
Header parent, KeyValueDB::Transaction t);
// Lookup leaf header for c oid
Header _lookup_header(const coll_t &cid, const ghobject_t &oid);
// Lookup header node for input
Header lookup_parent(Header input);
// Remove header and all related prefixes
int _clear(Header header, KeyValueDB::Transaction t);
// Adds to t operations necessary to add new_complete to the complete set
int merge_new_complete(Header header, const map<string, string> &new_complete,
GenericObjectMapIterator iter, KeyValueDB::Transaction t);
// Writes out State (mainly next_seq)
int write_state(KeyValueDB::Transaction _t);
// 0 if the complete set now contains all of key space, < 0 on error, 1 else
int need_parent(GenericObjectMapIterator iter);
// Copies header entry from parent @see rm_keys
int copy_up_header(Header header, KeyValueDB::Transaction t);
// Sets header @see set_header
void _set_header(Header header, const bufferlist &bl,
KeyValueDB::Transaction t);
};
WRITE_CLASS_ENCODER(GenericObjectMap::_Header)
WRITE_CLASS_ENCODER(GenericObjectMap::State)
#endif