diff --git a/.gitignore b/.gitignore index bca91c2..4a12d76 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,34 @@ +#Mac OS X Finder +.DS_Store + +# Xcode per user config +*.mode1 *.mode1v3 +*.mode2v3 +*.perspective +*.perspectivev3 *.pbxuser -build -*.framework +# Xcode 4 +xcuserdata/ +project.xcworkspace/ + +# Generated files +VersionX-revision.h + +# build products +build/ +*.[oa] + +# Other source repository archive directories +.hg +.svn +CVS +# automatic backup files +*~.nib +*.swp +*~ +*(Autosaved).rtfd/ +Backup[ ]of[ ]*.pages/ +Backup[ ]of[ ]*.key/ +Backup[ ]of[ ]*.numbers/ diff --git a/objc/NuBSON.m b/objc/NuBSON.m index b099218..3b099dd 100644 --- a/objc/NuBSON.m +++ b/objc/NuBSON.m @@ -40,10 +40,10 @@ void add_object_to_bson_buffer(bson_buffer *bb, id key, id object) case 'Q': bson_append_long(bb, name, [object longLongValue]); break; - case 'B': + case 'B': // C++/C99 bool + case 'c': // ObjC BOOL bson_append_bool(bb, name, [object boolValue]); break; - case 'c': case 'C': case 's': case 'S': @@ -82,6 +82,12 @@ void add_object_to_bson_buffer(bson_buffer *bb, id key, id object) else if ([object isKindOfClass:[NSData class]]) { bson_append_binary(bb, name, 0, [object bytes], [object length]); } + else if ([object isKindOfClass:[NSImage class]]) { + NSData *data = [object TIFFRepresentationUsingCompression:NSTIFFCompressionLZW factor:1.0L]; + if (data) { + bson_append_binary(bb, name, 0, [data bytes], [data length]); + } + } else if ([object isKindOfClass:[NuBSONObjectID class]]) { bson_append_oid(bb, name, [((NuBSONObjectID *) object) objectIDPointer]); } @@ -117,7 +123,26 @@ void add_object_to_bson_buffer(bson_buffer *bb, id key, id object) } } else if ([object respondsToSelector:@selector(cStringUsingEncoding:)]) { - bson_append_string(bb, name, [object cStringUsingEncoding:NSUTF8StringEncoding]); + // Check if we are dealing with an regular expression in the form of "//" + char *stringData = (char *)[object cStringUsingEncoding:NSUTF8StringEncoding]; + if (stringData[0] == '/') { // Quick check to see if we are dealing with a regex + NSArray *regexpComponents = [object componentsSeparatedByString:@"/"]; + NSString *expression = nil; + NSString *options = nil; + if ([regexpComponents count] == 3) { + expression = [regexpComponents objectAtIndex:1]; + options = [regexpComponents objectAtIndex:2]; + if (expression != nil) { + bson_append_regex(bb, name, [expression cStringUsingEncoding:NSUTF8StringEncoding],[options cStringUsingEncoding:NSUTF8StringEncoding]); + } + } + if (expression == nil) { // looks like we are dealing with a regular string + bson_append_string(bb, name, stringData); + } + } + else { + bson_append_string(bb, name, stringData); + } } else { NSLog(@"We have a problem. %@ cannot be serialized to bson", object); @@ -165,7 +190,7 @@ - (bson_oid_t) oid {return oid;} - (id) initWithData:(NSData *) data { - if (self = [super init]) { + if ((self = [super init])) { if ([data length] == 12) { memcpy(oid.bytes, [data bytes], 12); } @@ -178,7 +203,7 @@ - (id) copyWithZone:(NSZone *) zone return [[[self class] allocWithZone:zone] initWithObjectIDPointer:&oid]; } -- (NSInteger) hash { +- (NSUInteger) hash { return oid.ints[0] + oid.ints[1] + oid.ints[2]; } @@ -257,7 +282,7 @@ + (NuBSON *) bsonWithList:(id) list // internal, takes ownership of argument - (NuBSON *) initWithBSON:(bson) b { - if (self = [super init]) { + if ((self = [super init])) { bsonValue = b; } return self; @@ -315,6 +340,11 @@ - (void) dealloc { [super dealloc]; } +- (void) finalize { + bson_destroy(&bsonValue); + [super finalize]; +} + void dump_bson_iterator(bson_iterator it, const char *indent) { bson_iterator it2; @@ -335,7 +365,9 @@ void dump_bson_iterator(bson_iterator it, const char *indent) fprintf(stderr, "(int) %d\n", bson_iterator_int(&it)); break; case bson_string: + { fprintf(stderr, "(string) \"%s\"\n", bson_iterator_string(&it)); + } break; case bson_oid: bson_oid_to_string(bson_iterator_oid(&it), hex_oid); @@ -540,16 +572,16 @@ - (id) objectForKeyPath:(NSString *) keypath bson *bson_for_object(id object) { - bson *b = 0; + bson *b = malloc(sizeof(bson)); if (!object) { object = [NSDictionary dictionary]; } if ([object isKindOfClass:[NuBSON class]]) { - b = &(((NuBSON *)object)->bsonValue); + bson_copy(b,&(((NuBSON *)object)->bsonValue)); } else if ([object isKindOfClass:[NSDictionary class]]) { NuBSON *bsonObject = [[[NuBSON alloc] initWithDictionary:object] autorelease]; - b = &(bsonObject->bsonValue); + bson_copy(b,&(bsonObject->bsonValue)); // Needed for GC } else { NSLog(@"unable to convert objects of type %s to BSON (%@).", @@ -562,7 +594,7 @@ @implementation NuBSONBuffer - (id) init { - if (self = [super init]) { + if ((self = [super init])) { bson_buffer_init(& bb ); } return self; diff --git a/objc/NuMongoDB.h b/objc/NuMongoDB.h index c7b589d..f0596d1 100644 --- a/objc/NuMongoDB.h +++ b/objc/NuMongoDB.h @@ -21,6 +21,7 @@ limitations under the License. #include #include "bson.h" #include "mongo.h" +#include "gridfs.h" #import @@ -75,16 +76,20 @@ limitations under the License. - (NSMutableArray *) findArray:(id) query inCollection:(NSString *) collection; /*! Convenience method that returns search results as an array. */ - (NSMutableArray *) findArray:(id) query inCollection:(NSString *) collection returningFields:(id) fields numberToReturn:(int) nToReturn numberToSkip:(int) nToSkip; +/*! Convenience method that can be used to atomically modify a document (at most one) and return it. */ +- (NSMutableDictionary *) findAndModify:(id)collection options:(NSDictionary *)options inDatabase:(NSString *)database; /*! Convenience method that returns search results as a single object. */ - (NSMutableDictionary *) findOne:(id) query inCollection:(NSString *) collection; /*! Add an object to a collection, returning the _id of the new object. */ - (id) insertObject:(id) insert intoCollection:(NSString *) collection; +/*! Add an array of objects to a collection */ +- (void) insertObjects:(NSArray *)array intoCollection:(NSString *) collection; /*! Update an object in a collection. insertIfNecessary triggers an "upsert". */ - (void) updateObject:(id) update inCollection:(NSString *) collection withCondition:(id) condition insertIfNecessary:(BOOL) insertIfNecessary updateMultipleEntries:(BOOL) updateMultipleEntries; /*! Remove an object or objects matching a specified condition. */ - (void) removeWithCondition:(id) condition fromCollection:(NSString *) collection; /*! Count objects with a specified condition. */ -- (int) countWithCondition:(id) condition inCollection:(NSString *) collection inDatabase:(NSString *) database; +- (long long) countWithCondition:(id) condition inCollection:(NSString *) collection inDatabase:(NSString *) database; /*! Run an arbitrary database command. */ - (id) runCommand:(id) command inDatabase:(NSString *) database; /*! Get the names of the collections in a database. */ @@ -98,4 +103,9 @@ limitations under the License. /*! Close a database connection. */ - (void) close; +/*! GridFS write file */ +- (NSMutableDictionary *) writeFile:(NSString *)filePath withMIMEType:(NSString *)type inCollection:(NSString *) collection inDatabase:(NSString *) database; +- (NSMutableDictionary *) writeData:(NSData *)data named:(NSString *)file withMIMEType:(NSString *)type inCollection:(NSString *) collection inDatabase:(NSString *) database; +- (NSData *) retrieveDataforGridFSFile:(NSString *) filePath inCollection:(NSString *) collection inDatabase:(NSString *) database; +-(BOOL) removeFile:(NSString *)filePath inCollection:(NSString *) collection inDatabase:(NSString *) database; @end diff --git a/objc/NuMongoDB.m b/objc/NuMongoDB.m index ef66cbd..d2f17c9 100644 --- a/objc/NuMongoDB.m +++ b/objc/NuMongoDB.m @@ -46,6 +46,12 @@ - (void) dealloc [super dealloc]; } +- (void) finalize +{ + mongo_cursor_destroy(cursor); + [super finalize]; +} + - (NSMutableArray *) arrayValue { NSMutableArray *result = [NSMutableArray array]; @@ -114,6 +120,7 @@ - (NuMongoDBCursor *) find:(id) query inCollection:(NSString *) collection { bson *b = bson_for_object(query); mongo_cursor *cursor = mongo_find(conn, [collection cStringUsingEncoding:NSUTF8StringEncoding], b, 0, 0, 0, 0 ); + bson_destroy(b); return [[[NuMongoDBCursor alloc] initWithCursor:cursor] autorelease]; } @@ -122,6 +129,8 @@ - (NuMongoDBCursor *) find:(id) query inCollection:(NSString *) collection retur bson *b = bson_for_object(query); bson *f = bson_for_object(fields); mongo_cursor *cursor = mongo_find(conn, [collection cStringUsingEncoding:NSUTF8StringEncoding], b, f, nToReturn, nToSkip, 0 ); + bson_destroy(b); + bson_destroy(f); return [[[NuMongoDBCursor alloc] initWithCursor:cursor] autorelease]; } @@ -142,6 +151,22 @@ - (NSMutableDictionary *) findOne:(id) query inCollection:(NSString *) collectio bson *b = bson_for_object(query); bson bsonResult; bson_bool_t result = mongo_find_one(conn, [collection cStringUsingEncoding:NSUTF8StringEncoding], b, 0, &bsonResult); + bson_destroy(b); + return result ? [[[[NuBSON alloc] initWithBSON:bsonResult] autorelease] dictionaryValue] : nil; +} + +- (NSMutableDictionary *) findAndModify:(id)collection options:(NSDictionary *)options inDatabase:(NSString *)database +{ + NuBSONBuffer *bsonBuffer = [[[NuBSONBuffer alloc] init] autorelease]; + [bsonBuffer addObject:collection withKey:@"findandmodify"]; + id keys = [options allKeys]; + for (int i = 0; i < [keys count]; i++) { + id key = [keys objectAtIndex:i]; + [bsonBuffer addObject:[options objectForKey:key] withKey:key]; + } + + bson bsonResult; + bson_bool_t result = mongo_run_command(conn, [database cStringUsingEncoding:NSUTF8StringEncoding], &([bsonBuffer bsonValue]->bsonValue), &bsonResult); return result ? [[[[NuBSON alloc] initWithBSON:bsonResult] autorelease] dictionaryValue] : nil; } @@ -154,6 +179,7 @@ - (id) insertObject:(id) insert intoCollection:(NSString *) collection bson *b = bson_for_object(insert); if (b) { mongo_insert(conn, [collection cStringUsingEncoding:NSUTF8StringEncoding], b); + bson_destroy(b); return [insert objectForKey:@"_id"]; } else { @@ -162,6 +188,34 @@ - (id) insertObject:(id) insert intoCollection:(NSString *) collection } } +- (void) insertObjects:(NSArray *) array intoCollection:(NSString *) collection +{ + int count = [array count]; + bson bArray[count]; + bson *bpArray[count]; + for (int i = 0; i < count; i++) { + bpArray[i] = &bArray[i]; + } + + NSMutableArray *BSONObjects = [NSMutableArray array]; + for (int i = 0; i < count; i++) { + id insert = [array objectAtIndex:i]; + if (![insert objectForKey:@"_id"]) { + insert = [[insert mutableCopy] autorelease]; + [insert setObject:[NuBSONObjectID objectID] forKey:@"_id"]; + } + NuBSON *bsonObject = [NuBSON bsonWithDictionary:insert]; + [BSONObjects addObject:bsonObject]; + bson_copy(&bArray[i],&bsonObject->bsonValue); + } + + mongo_insert_batch(conn, [collection cStringUsingEncoding:NSUTF8StringEncoding], bpArray, count); + // Free objects + for (int i = 0; i < count; i++) { + bson_destroy(&bArray[i]); + } +} + - (void) updateObject:(id) update inCollection:(NSString *) collection withCondition:(id) condition insertIfNecessary:(BOOL) insertIfNecessary updateMultipleEntries:(BOOL) updateMultipleEntries { @@ -172,6 +226,8 @@ - (void) updateObject:(id) update inCollection:(NSString *) collection bcondition, bupdate, (insertIfNecessary ? MONGO_UPDATE_UPSERT : 0) + (updateMultipleEntries ? MONGO_UPDATE_MULTI : 0)); + bson_destroy(bupdate); + bson_destroy(bcondition); } else { NSLog(@"incomplete update: update and condition must not be nil."); @@ -182,9 +238,10 @@ - (void) removeWithCondition:(id) condition fromCollection:(NSString *) collecti { bson *bcondition = bson_for_object(condition); mongo_remove(conn, [collection cStringUsingEncoding:NSUTF8StringEncoding], bcondition); + bson_destroy(bcondition); } -- (int) countWithCondition:(id) condition inCollection:(NSString *) collection inDatabase:(NSString *) database +- (long long) countWithCondition:(id) condition inCollection:(NSString *) collection inDatabase:(NSString *) database { bson *bcondition = bson_for_object(condition); return mongo_count(conn, [database cStringUsingEncoding:NSUTF8StringEncoding], [collection cStringUsingEncoding:NSUTF8StringEncoding], bcondition); @@ -195,6 +252,7 @@ - (id) runCommand:(id) command inDatabase:(NSString *) database bson *bcommand = bson_for_object(command); bson bsonResult; bson_bool_t result = mongo_run_command(conn, [database cStringUsingEncoding:NSUTF8StringEncoding], bcommand, &bsonResult); + bson_destroy(bcommand); return result ? [[[[NuBSON alloc] initWithBSON:bsonResult] autorelease] dictionaryValue] : nil; } @@ -247,4 +305,80 @@ - (void) close mongo_destroy(conn ); } + +- (NSMutableDictionary *) writeFile:(NSString *)filePath withMIMEType:(NSString *)type inCollection:(NSString *) collection inDatabase:(NSString *) database +{ + bson output; + gridfs gfs[1]; + + gridfs_init(conn, [database cStringUsingEncoding:NSUTF8StringEncoding], [collection cStringUsingEncoding:NSUTF8StringEncoding], gfs); + output = gridfs_store_file(gfs, [filePath cStringUsingEncoding:NSUTF8StringEncoding], [filePath cStringUsingEncoding:NSUTF8StringEncoding], [type cStringUsingEncoding:NSUTF8StringEncoding]); + gridfs_destroy(gfs); + + return [[[[NuBSON alloc] initWithBSON:output] autorelease] dictionaryValue]; +} + +- (NSMutableDictionary *) writeData:(NSData *)data named:(NSString *)file withMIMEType:(NSString *)type inCollection:(NSString *) collection inDatabase:(NSString *) database +{ + bson output; + gridfs gfs[1]; + gridfile gfile[1]; + char buffer[1024]; + int n; + NSUInteger i = 0; + + gridfs_init(conn, [database cStringUsingEncoding:NSUTF8StringEncoding], [collection cStringUsingEncoding:NSUTF8StringEncoding], gfs); + gridfile_writer_init( gfile, gfs, [file cStringUsingEncoding:NSUTF8StringEncoding], [type cStringUsingEncoding:NSUTF8StringEncoding]); + while(data.length-i > 0) { + n = MIN(data.length-i,1024); + [data getBytes:buffer range:NSMakeRange(i,n)]; + gridfile_write_buffer(gfile, buffer, n); + i += n; + } + output = gridfile_writer_done(gfile); + gridfs_destroy(gfs); + + return [[[[NuBSON alloc] initWithBSON:output] autorelease] dictionaryValue]; +} + + +- (NSData *) retrieveDataforGridFSFile:(NSString *) filePath inCollection:(NSString *) collection inDatabase:(NSString *) database +{ + gridfs gfs[1]; + gridfile gfile[1]; + gridfs_offset length, chunkLength; + NSUInteger chunkSize, numChunks; + + gridfs_init(conn, [database cStringUsingEncoding:NSUTF8StringEncoding], [collection cStringUsingEncoding:NSUTF8StringEncoding], gfs); + if (gridfs_find_filename(gfs, [filePath cStringUsingEncoding:NSUTF8StringEncoding], gfile)) { + length = gridfile_get_contentlength(gfile); + chunkSize = gridfile_get_chunksize(gfile); + numChunks = gridfile_get_numchunks(gfile); + NSMutableData *data = [NSMutableData dataWithCapacity:(NSUInteger)length]; + + char buffer[chunkSize]; + + for (NSUInteger i = 0; i < numChunks; i++) { + chunkLength = gridfile_read(gfile, chunkSize, buffer); + [data appendBytes:buffer length:(NSUInteger)chunkLength]; + } + gridfs_destroy(gfs); + + return data; + } + else { + return nil; + } +} + +-(BOOL) removeFile:(NSString *)filePath inCollection:(NSString *) collection inDatabase:(NSString *) database +{ + gridfs gfs[1]; + gridfs_init(conn, [database cStringUsingEncoding:NSUTF8StringEncoding], [collection cStringUsingEncoding:NSUTF8StringEncoding], gfs); + gridfs_remove_filename(gfs, [filePath cStringUsingEncoding:NSUTF8StringEncoding]); + gridfs_destroy(gfs); + + return YES; +} + @end diff --git a/src/bson.c b/src/bson.c index a12276b..59f0a8c 100644 --- a/src/bson.c +++ b/src/bson.c @@ -129,8 +129,8 @@ time_t bson_oid_generated_time(bson_oid_t* oid){ time_t out; bson_big_endian32(&out, &oid->ints[0]); return out; -} +} void bson_print( bson * b ){ bson_print_raw( b->data , 0 ); } @@ -139,6 +139,7 @@ void bson_print_raw( const char * data , int depth ){ bson_iterator i; const char * key; int temp; + bson_timestamp_t ts; char oidhex[25]; bson_iterator_init( &i , data ); @@ -147,7 +148,7 @@ void bson_print_raw( const char * data , int depth ){ if ( t == 0 ) break; key = bson_iterator_key( &i ); - + for ( temp=0; temp<=depth; temp++ ) printf( "\t" ); printf( "%s : %d \t " , key , t ); @@ -158,6 +159,10 @@ void bson_print_raw( const char * data , int depth ){ case bson_string: printf( "%s" , bson_iterator_string( &i ) ); break; case bson_null: printf( "null" ); break; case bson_oid: bson_oid_to_string(bson_iterator_oid(&i), oidhex); printf( "%s" , oidhex ); break; + case bson_timestamp: + ts = bson_iterator_timestamp( &i ); + printf("i: %d, t: %d", ts.i, ts.t); + break; case bson_object: case bson_array: printf( "\n" ); @@ -306,6 +311,13 @@ int64_t bson_iterator_long( const bson_iterator * i ){ } } +bson_timestamp_t bson_iterator_timestamp( const bson_iterator * i){ + bson_timestamp_t ts; + bson_little_endian32(&(ts.i), bson_iterator_value(i)); + bson_little_endian32(&(ts.t), bson_iterator_value(i) + 4); + return ts; +} + bson_bool_t bson_iterator_bool( const bson_iterator * i ){ switch (bson_iterator_type(i)){ case bson_bool: return bson_iterator_bool_raw(i); @@ -353,14 +365,18 @@ time_t bson_iterator_time_t(const bson_iterator * i){ } int bson_iterator_bin_len( const bson_iterator * i ){ - return bson_iterator_int_raw( i ); + return (bson_iterator_bin_type(i) == 2) + ? bson_iterator_int_raw( i ) - 4 + : bson_iterator_int_raw( i ); } char bson_iterator_bin_type( const bson_iterator * i ){ return bson_iterator_value(i)[4]; } const char * bson_iterator_bin_data( const bson_iterator * i ){ - return bson_iterator_value( i ) + 5; + return (bson_iterator_bin_type( i ) == 2) + ? bson_iterator_value( i ) + 9 + : bson_iterator_value( i ) + 5; } const char * bson_iterator_regex( const bson_iterator * i ){ @@ -520,10 +536,19 @@ bson_buffer * bson_append_code_w_scope( bson_buffer * b , const char * name , co } bson_buffer * bson_append_binary( bson_buffer * b, const char * name, char type, const char * str, int len ){ - if ( ! bson_append_estart( b , bson_bindata , name , 4+1+len ) ) return 0; - bson_append32(b, &len); - bson_append_byte(b, type); - bson_append(b, str, len); + if ( type == 2 ){ + int subtwolen = len + 4; + if ( ! bson_append_estart( b , bson_bindata , name , 4+1+4+len ) ) return 0; + bson_append32(b, &subtwolen); + bson_append_byte(b, type); + bson_append32(b, &len); + bson_append(b, str, len); + }else{ + if ( ! bson_append_estart( b , bson_bindata , name , 4+1+len ) ) return 0; + bson_append32(b, &len); + bson_append_byte(b, type); + bson_append(b, str, len); + } return b; } bson_buffer * bson_append_oid( bson_buffer * b , const char * name , const bson_oid_t * oid ){ @@ -563,15 +588,23 @@ bson_buffer * bson_append_element( bson_buffer * b, const char * name_or_null, c bson_ensure_space(b, size); bson_append(b, elem->cur, size); }else{ - int data_size = size - 1 - strlen(bson_iterator_key(elem)); + int data_size = size - 2 - strlen(bson_iterator_key(elem)); bson_append_estart(b, elem->cur[0], name_or_null, data_size); - bson_append(b, name_or_null, strlen(name_or_null)); bson_append(b, bson_iterator_value(elem), data_size); } return b; } +bson_buffer * bson_append_timestamp( bson_buffer * b, const char * name, bson_timestamp_t * ts ){ + if ( ! bson_append_estart( b , bson_timestamp , name , 8 ) ) return 0; + + bson_append32( b , &(ts->i) ); + bson_append32( b , &(ts->t) ); + + return b; +} + bson_buffer * bson_append_date( bson_buffer * b , const char * name , bson_date_t millis ){ if ( ! bson_append_estart( b , bson_date , name , 8 ) ) return 0; bson_append64( b , &millis ); @@ -601,7 +634,7 @@ bson_buffer * bson_append_finish_object( bson_buffer * b ){ int i; if ( ! bson_ensure_space( b , 1 ) ) return 0; bson_append_byte( b , 0 ); - + start = b->buf + b->stack[ --b->stackPos ]; i = b->cur - start; bson_little_endian32(start, &i); @@ -615,6 +648,12 @@ void* bson_malloc(int size){ return p; } +void* bson_realloc(void* ptr, int size){ + void* p = realloc(ptr, size); + bson_fatal_msg(!!p, "realloc() failed"); + return p; +} + static bson_err_handler err_handler = NULL; bson_err_handler set_bson_err_handler(bson_err_handler func){ diff --git a/src/bson.h b/src/bson.h index a2f0bf0..2c5934f 100644 --- a/src/bson.h +++ b/src/bson.h @@ -75,11 +75,15 @@ typedef union{ typedef int64_t bson_date_t; /* milliseconds since epoch UTC */ +typedef struct { + int i; /* increment */ + int t; /* time in seconds */ +} bson_timestamp_t; + /* ---------------------------- READING ------------------------------ */ - bson * bson_empty(bson * obj); /* returns pointer to static empty bson object */ void bson_copy(bson* out, const bson* in); /* puts data in new buffer. NOOP if out==NULL */ bson * bson_from_buffer(bson * b, bson_buffer * buf); @@ -109,6 +113,9 @@ double bson_iterator_double( const bson_iterator * i ); int bson_iterator_int( const bson_iterator * i ); int64_t bson_iterator_long( const bson_iterator * i ); +/* return the bson timestamp as a whole or in parts */ +bson_timestamp_t bson_iterator_timestamp( const bson_iterator * i ); + /* false: boolean false, 0 in any type, or null */ /* true: anything else (even empty strings and objects) */ bson_bool_t bson_iterator_bool( const bson_iterator * i ); @@ -182,6 +189,7 @@ bson_buffer * bson_append_undefined( bson_buffer * b , const char * name ); bson_buffer * bson_append_regex( bson_buffer * b , const char * name , const char * pattern, const char * opts ); bson_buffer * bson_append_bson( bson_buffer * b , const char * name , const bson* bson); bson_buffer * bson_append_element( bson_buffer * b, const char * name_or_null, const bson_iterator* elem); +bson_buffer * bson_append_timestamp( bson_buffer * b, const char * name, bson_timestamp_t * ts ); /* these both append a bson_date */ bson_buffer * bson_append_date(bson_buffer * b, const char * name, bson_date_t millis); @@ -200,6 +208,7 @@ void bson_incnumstr(char* str); ------------------------------ */ void * bson_malloc(int size); /* checks return value */ +void * bson_realloc(void * ptr, int size); /* checks return value */ /* bson_err_handlers shouldn't return!!! */ typedef void(*bson_err_handler)(const char* errmsg); @@ -208,8 +217,6 @@ typedef void(*bson_err_handler)(const char* errmsg); /* default handler prints error then exits with failure*/ bson_err_handler set_bson_err_handler(bson_err_handler func); - - /* does nothing is ok != 0 */ void bson_fatal( int ok ); void bson_fatal_msg( int ok, const char* msg ); diff --git a/src/gridfs.c b/src/gridfs.c new file mode 100644 index 0000000..0494fa8 --- /dev/null +++ b/src/gridfs.c @@ -0,0 +1,799 @@ +/*--------------------------------------------------------------------*/ +/* gridfs.c */ +/* Author: Christopher Triolo */ +/*--------------------------------------------------------------------*/ + +#include "gridfs.h" +#include "mongo.h" +#include "bson.h" +#include +#include +#include +#include +#define TRUE 1 +#define FALSE 0 + + +/*--------------------------------------------------------------------*/ + +static bson * chunk_new(bson_oid_t id, int chunkNumber, + const char * data, int len) + +{ + bson * b; + bson_buffer buf; + + b = (bson *)bson_malloc(sizeof(bson)); + if (b == NULL) return NULL; + + bson_buffer_init(&buf); + bson_append_oid(&buf, "files_id", &id); + bson_append_int(&buf, "n", chunkNumber); + bson_append_binary(&buf, "data", 2, data, len); + bson_from_buffer(b, &buf); + return b; +} + +/*--------------------------------------------------------------------*/ + +static void chunk_free(bson * oChunk) + +{ + bson_destroy(oChunk); + free(oChunk); +} + +/*--------------------------------------------------------------------*/ + +int gridfs_init(mongo_connection * client, const char * dbname, + const char * prefix, gridfs* gfs) +{ + int options; + bson_buffer bb; + bson b; + bson out; + bson_bool_t success; + + gfs->client = client; + + /* Allocate space to own the dbname */ + gfs->dbname = (const char *)bson_malloc(strlen(dbname)+1); + if (gfs->dbname == NULL) { + return FALSE; + } + strcpy((char*)gfs->dbname, dbname); + + /* Allocate space to own the prefix */ + if (prefix == NULL) prefix = "fs"; + gfs->prefix = (const char *)bson_malloc(strlen(prefix)+1); + if (gfs->prefix == NULL) { + free((char*)gfs->dbname); + return FALSE; + } + strcpy((char *)gfs->prefix, prefix); + + /* Allocate space to own files_ns */ + gfs->files_ns = + (const char *) bson_malloc (strlen(prefix)+strlen(dbname)+strlen(".files")+2); + if (gfs->files_ns == NULL) { + free((char*)gfs->dbname); + free((char*)gfs->prefix); + return FALSE; + } + strcpy((char*)gfs->files_ns, dbname); + strcat((char*)gfs->files_ns, "."); + strcat((char*)gfs->files_ns, prefix); + strcat((char*)gfs->files_ns, ".files"); + + /* Allocate space to own chunks_ns */ + gfs->chunks_ns = (const char *) bson_malloc(strlen(prefix) + strlen(dbname) + + strlen(".chunks") + 2); + if (gfs->chunks_ns == NULL) { + free((char*)gfs->dbname); + free((char*)gfs->prefix); + free((char*)gfs->files_ns); + return FALSE; + } + strcpy((char*)gfs->chunks_ns, dbname); + strcat((char*)gfs->chunks_ns, "."); + strcat((char*)gfs->chunks_ns, prefix); + strcat((char*)gfs->chunks_ns, ".chunks"); + + bson_buffer_init(&bb); + bson_append_int(&bb, "filename", 1); + bson_from_buffer(&b, &bb); + options = 0; + success = mongo_create_index(gfs->client, gfs->files_ns, &b, options, &out); + bson_destroy(&b); + if (!success) { + free((char*)gfs->dbname); + free((char*)gfs->prefix); + free((char*)gfs->files_ns); + free((char*)gfs->chunks_ns); + return FALSE; + } + + bson_buffer_init(&bb); + bson_append_int(&bb, "files_id", 1); + bson_append_int(&bb, "n", 1); + bson_from_buffer(&b, &bb); + options = MONGO_INDEX_UNIQUE; + success = mongo_create_index(gfs->client, gfs->chunks_ns, &b, options, &out); + bson_destroy(&b); + if (!success) { + free((char*)gfs->dbname); + free((char*)gfs->prefix); + free((char*)gfs->files_ns); + free((char*)gfs->chunks_ns); + return FALSE; + } + + return TRUE; +} + +/*--------------------------------------------------------------------*/ + +void gridfs_destroy(gridfs* gfs) + +{ + if (gfs == NULL) return; + if (gfs->dbname) free((char*)gfs->dbname); + if (gfs->prefix) free((char*)gfs->prefix); + if (gfs->files_ns) free((char*)gfs->files_ns); + if (gfs->chunks_ns) free((char*)gfs->chunks_ns); +} + +/*--------------------------------------------------------------------*/ + +static bson gridfs_insert_file( gridfs* gfs, const char* name, + const bson_oid_t id, gridfs_offset length, + const char* contenttype) +{ + bson command; + bson res; + bson ret; + bson_buffer buf; + bson_iterator it; + + /* Check run md5 */ + bson_buffer_init(&buf); + bson_append_oid(&buf, "filemd5", &id); + bson_append_string(&buf, "root", gfs->prefix); + bson_from_buffer(&command, &buf); + assert(mongo_run_command(gfs->client, gfs->dbname, + &command, &res)); + bson_destroy(&command); + + /* Create and insert BSON for file metadata */ + bson_buffer_init(&buf); + bson_append_oid(&buf, "_id", &id); + if (name != NULL && *name != '\0') { + bson_append_string(&buf, "filename", name); + } + bson_append_long(&buf, "length", length); + bson_append_int(&buf, "chunkSize", DEFAULT_CHUNK_SIZE); + bson_append_date(&buf, "uploadDate", (bson_date_t)1000*time(NULL)); + bson_find(&it, &res, "md5"); + bson_append_string(&buf, "md5", bson_iterator_string(&it)); + bson_destroy(&res); + if (contenttype != NULL && *contenttype != '\0') { + bson_append_string(&buf, "contentType", contenttype); + } + bson_from_buffer(&ret, &buf); + mongo_insert(gfs->client, gfs->files_ns, &ret); + + return ret; +} + +/*--------------------------------------------------------------------*/ + +bson gridfs_store_buffer( gridfs* gfs, const char* data, + gridfs_offset length, const char* remotename, + const char * contenttype) + +{ + char const * const end = data + length; + bson_oid_t id; + int chunkNumber = 0; + int chunkLen; + bson * oChunk; + + /* Large files Assertion */ + assert(length <= 0xffffffff); + + /* Generate and append an oid*/ + bson_oid_gen(&id); + + /* Insert the file's data chunk by chunk */ + while (data < end) { + chunkLen = DEFAULT_CHUNK_SIZE < (unsigned int)(end - data) ? + DEFAULT_CHUNK_SIZE : (unsigned int)(end - data); + oChunk = chunk_new( id, chunkNumber, data, chunkLen ); + mongo_insert(gfs->client, gfs->chunks_ns, oChunk); + chunk_free(oChunk); + chunkNumber++; + data += chunkLen; + } + + /* Inserts file's metadata */ + return gridfs_insert_file(gfs, remotename, id, length, contenttype); +} + +/*--------------------------------------------------------------------*/ + +void gridfile_writer_init( gridfile* gfile, gridfs* gfs, + const char* remote_name, const char* content_type ) +{ + gfile->gfs = gfs; + + bson_oid_gen( &(gfile->id) ); + gfile->chunk_num = 0; + gfile->length = 0; + gfile->pending_len = 0; + gfile->pending_data = NULL; + + gfile->remote_name = (const char *)bson_malloc( strlen( remote_name ) + 1 ); + strcpy( (char *)gfile->remote_name, remote_name ); + + gfile->content_type = (const char *)bson_malloc( strlen( content_type ) ); + strcpy( (char *)gfile->content_type, content_type ); +} + +/*--------------------------------------------------------------------*/ + +void gridfile_write_buffer( gridfile* gfile, const char* data, gridfs_offset length ) +{ + + int bytes_left = 0; + int data_partial_len = 0; + int chunks_to_write = 0; + char* buffer; + bson* oChunk; + gridfs_offset to_write = length + gfile->pending_len; + + if ( to_write < DEFAULT_CHUNK_SIZE ) { /* Less than one chunk to write */ + if( gfile->pending_data ) { + gfile->pending_data = (char *)bson_realloc((void *)gfile->pending_data, gfile->pending_len + to_write); + memcpy( gfile->pending_data + gfile->pending_len, data, length ); + } else if (to_write > 0) { + gfile->pending_data = (char *)bson_malloc(to_write); + memcpy( gfile->pending_data, data, length ); + } + gfile->pending_len += length; + + } else { /* At least one chunk of data to write */ + + /* If there's a pending chunk to be written, we need to combine + * the buffer provided up to DEFAULT_CHUNK_SIZE. + */ + if ( gfile->pending_len > 0 ) { + chunks_to_write = to_write / DEFAULT_CHUNK_SIZE; + bytes_left = to_write % DEFAULT_CHUNK_SIZE; + + data_partial_len = DEFAULT_CHUNK_SIZE - gfile->pending_len; + buffer = (char *)bson_malloc( DEFAULT_CHUNK_SIZE ); + memcpy(buffer, gfile->pending_data, gfile->pending_len); + memcpy(buffer + gfile->pending_len, data, data_partial_len); + + oChunk = chunk_new(gfile->id, gfile->chunk_num, buffer, DEFAULT_CHUNK_SIZE); + mongo_insert(gfile->gfs->client, gfile->gfs->chunks_ns, oChunk); + chunk_free(oChunk); + gfile->chunk_num++; + gfile->length += DEFAULT_CHUNK_SIZE; + data += data_partial_len; + + chunks_to_write--; + + free(buffer); + } + + while( chunks_to_write > 0 ) { + oChunk = chunk_new(gfile->id, gfile->chunk_num, data, DEFAULT_CHUNK_SIZE); + mongo_insert(gfile->gfs->client, gfile->gfs->chunks_ns, oChunk); + chunk_free(oChunk); + gfile->chunk_num++; + chunks_to_write--; + gfile->length += DEFAULT_CHUNK_SIZE; + data += DEFAULT_CHUNK_SIZE; + } + + free(gfile->pending_data); + + /* If there are any leftover bytes, store them as pending data. */ + if( bytes_left == 0 ) + gfile->pending_data = NULL; + else { + gfile->pending_data = (char *)bson_malloc( bytes_left ); + memcpy( gfile->pending_data, data, bytes_left ); + } + + gfile->pending_len = bytes_left; + } +} + +/*--------------------------------------------------------------------*/ + +bson gridfile_writer_done( gridfile* gfile ) +{ + + /* write any remaining pending chunk data. + * pending data will always take up less than one chunk + */ + bson* oChunk; + if( gfile->pending_data ) + { + oChunk = chunk_new(gfile->id, gfile->chunk_num, gfile->pending_data, gfile->pending_len); + mongo_insert(gfile->gfs->client, gfile->gfs->chunks_ns, oChunk); + chunk_free(oChunk); + free(gfile->pending_data); + gfile->length += gfile->pending_len; + } + + /* insert into files collection */ + return gridfs_insert_file(gfile->gfs, gfile->remote_name, gfile->id, + gfile->length, gfile->content_type); +} + +/*--------------------------------------------------------------------*/ + +bson gridfs_store_file(gridfs* gfs, const char* filename, + const char* remotename, const char* contenttype) +{ + char buffer[DEFAULT_CHUNK_SIZE]; + FILE * fd; + bson_oid_t id; + int chunkNumber = 0; + gridfs_offset length = 0; + gridfs_offset chunkLen = 0; + bson* oChunk; + + /* Open the file and the correct stream */ + if (strcmp(filename, "-") == 0) fd = stdin; + else fd = fopen(filename, "rb"); + assert(fd != NULL); /* No such file */ + + /* Generate and append an oid*/ + bson_oid_gen(&id); + + /* Insert the file chunk by chunk */ + chunkLen = fread(buffer, 1, DEFAULT_CHUNK_SIZE, fd); + do { + oChunk = chunk_new( id, chunkNumber, buffer, chunkLen ); + mongo_insert(gfs->client, gfs->chunks_ns, oChunk); + chunk_free(oChunk); + length += chunkLen; + chunkNumber++; + chunkLen = fread(buffer, 1, DEFAULT_CHUNK_SIZE, fd); + } while (chunkLen != 0); + + /* Close the file stream */ + if (fd != stdin) fclose(fd); + + /* Large files Assertion */ + /* assert(length <= 0xffffffff); */ + + /* Optional Remote Name */ + if (remotename == NULL || *remotename == '\0') { + remotename = filename; } + + /* Inserts file's metadata */ + return gridfs_insert_file(gfs, remotename, id, length, contenttype); +} + +/*--------------------------------------------------------------------*/ + +void gridfs_remove_filename(gridfs* gfs, const char* filename ) + +{ + bson query; + bson_buffer buf; + mongo_cursor* files; + bson file; + bson_iterator it; + bson_oid_t id; + bson b; + + bson_buffer_init(&buf); + bson_append_string(&buf, "filename", filename); + bson_from_buffer(&query, &buf); + files = mongo_find(gfs->client, gfs->files_ns, &query, NULL, 0, 0, 0); + bson_destroy(&query); + + /* Remove each file and it's chunks from files named filename */ + while (mongo_cursor_next(files)) { + file = files->current; + bson_find(&it, &file, "_id"); + id = *bson_iterator_oid(&it); + + /* Remove the file with the specified id */ + bson_buffer_init(&buf); + bson_append_oid(&buf, "_id", &id); + bson_from_buffer(&b, &buf); + mongo_remove( gfs->client, gfs->files_ns, &b); + bson_destroy(&b); + + /* Remove all chunks from the file with the specified id */ + bson_buffer_init(&buf); + bson_append_oid(&buf, "files_id", &id); + bson_from_buffer(&b, &buf); + mongo_remove( gfs->client, gfs->chunks_ns, &b); + bson_destroy(&b); + } + +} + +/*--------------------------------------------------------------------*/ + +int gridfs_find_query(gridfs* gfs, bson* query, + gridfile* gfile ) + +{ + bson_buffer date_buffer; + bson uploadDate; + bson_buffer buf; + bson finalQuery; + bson out; + int i; + + bson_buffer_init(&date_buffer); + bson_append_int(&date_buffer, "uploadDate", -1); + bson_from_buffer(&uploadDate, &date_buffer); + bson_buffer_init(&buf); + bson_append_bson(&buf, "query", query); + bson_append_bson(&buf, "orderby", &uploadDate); + bson_from_buffer(&finalQuery, &buf); + + + i = (mongo_find_one(gfs->client, gfs->files_ns, + &finalQuery, NULL, &out)); + bson_destroy(&uploadDate); + bson_destroy(&finalQuery); + if (!i) + return FALSE; + else { + gridfile_init(gfs, &out, gfile); + bson_destroy(&out); + return TRUE; + } +} + +/*--------------------------------------------------------------------*/ + +int gridfs_find_filename(gridfs* gfs, const char* filename, + gridfile* gfile) + +{ + bson query; + bson_buffer buf; + int i; + + bson_buffer_init(&buf); + bson_append_string(&buf, "filename", filename); + bson_from_buffer(&query, &buf) ; + i = gridfs_find_query(gfs, &query, gfile); + bson_destroy(&query); + return i; +} + +/*--------------------------------------------------------------------*/ + +int gridfile_init(gridfs* gfs, bson* meta, gridfile* gfile) + +{ + gfile->gfs = gfs; + gfile->pos = 0; + gfile->meta = (bson*)bson_malloc(sizeof(bson)); + if (gfile->meta == NULL) return FALSE; + bson_copy(gfile->meta, meta); + return TRUE; +} + +/*--------------------------------------------------------------------*/ + +void gridfile_destroy(gridfile* gfile) + +{ + bson_destroy(gfile->meta); + free(gfile->meta); +} + +/*--------------------------------------------------------------------*/ + +bson_bool_t gridfile_exists(gridfile* gfile) + +{ + return (bson_bool_t)(gfile != NULL || gfile->meta == NULL); +} + +/*--------------------------------------------------------------------*/ + +const char* gridfile_get_filename(gridfile* gfile) + +{ + bson_iterator it; + + bson_find(&it, gfile->meta, "filename"); + return bson_iterator_string(&it); +} + +/*--------------------------------------------------------------------*/ + +int gridfile_get_chunksize(gridfile* gfile) + +{ + bson_iterator it; + + bson_find(&it, gfile->meta, "chunkSize"); + return bson_iterator_int(&it); +} + +/*--------------------------------------------------------------------*/ + +gridfs_offset gridfile_get_contentlength(gridfile* gfile) + +{ + bson_iterator it; + + bson_find(&it, gfile->meta, "length"); + + if( bson_iterator_type( &it ) == bson_int ) + return (gridfs_offset)bson_iterator_int( &it ); + else + return (gridfs_offset)bson_iterator_long( &it ); +} + +/*--------------------------------------------------------------------*/ + +const char *gridfile_get_contenttype(gridfile* gfile) + +{ + bson_iterator it; + + if (bson_find(&it, gfile->meta, "contentType")) + return bson_iterator_string( &it ); + else return NULL; +} + +/*--------------------------------------------------------------------*/ + +bson_date_t gridfile_get_uploaddate(gridfile* gfile) + +{ + bson_iterator it; + + bson_find(&it, gfile->meta, "uploadDate"); + return bson_iterator_date( &it ); +} + +/*--------------------------------------------------------------------*/ + +const char* gridfile_get_md5(gridfile* gfile) + +{ + bson_iterator it; + + bson_find(&it, gfile->meta, "md5"); + return bson_iterator_string( &it ); +} + +/*--------------------------------------------------------------------*/ + +const char* gridfile_get_field(gridfile* gfile, const char* name) + +{ + bson_iterator it; + + bson_find(&it, gfile->meta, name); + return bson_iterator_value( &it ); +} + +/*--------------------------------------------------------------------*/ + +bson_bool_t gridfile_get_boolean(gridfile* gfile, const char* name) +{ + bson_iterator it; + + bson_find(&it, gfile->meta, name); + return bson_iterator_bool( &it ); +} + +/*--------------------------------------------------------------------*/ +bson gridfile_get_metadata(gridfile* gfile) + +{ + bson sub; + bson_iterator it; + + if (bson_find(&it, gfile->meta, "metadata")) { + bson_iterator_subobject( &it, &sub ); + return sub; + } + else { + bson_empty(&sub); + return sub; + } +} + +/*--------------------------------------------------------------------*/ + +int gridfile_get_numchunks(gridfile* gfile) + +{ + bson_iterator it; + gridfs_offset length; + gridfs_offset chunkSize; + double numchunks; + + bson_find(&it, gfile->meta, "length"); + + if( bson_iterator_type( &it ) == bson_int ) + length = (gridfs_offset)bson_iterator_int( &it ); + else + length = (gridfs_offset)bson_iterator_long( &it ); + + bson_find(&it, gfile->meta, "chunkSize"); + chunkSize = bson_iterator_int(&it); + numchunks = ((double)length/(double)chunkSize); + return (numchunks - (int)numchunks > 0) + ? (int)(numchunks+1) + : (int)(numchunks); +} + +/*--------------------------------------------------------------------*/ + +bson gridfile_get_chunk(gridfile* gfile, int n) + +{ + bson query; + bson out; + bson_buffer buf; + bson_iterator it; + bson_oid_t id; + + bson_buffer_init(&buf); + bson_find(&it, gfile->meta, "_id"); + id = *bson_iterator_oid(&it); + bson_append_oid(&buf, "files_id", &id); + bson_append_int(&buf, "n", n); + bson_from_buffer(&query, &buf); + + assert(mongo_find_one(gfile->gfs->client, + gfile->gfs->chunks_ns, + &query, NULL, &out)); + return out; +} + +/*--------------------------------------------------------------------*/ + +mongo_cursor* gridfile_get_chunks(gridfile* gfile, int start, int size) + +{ + bson_iterator it; + bson_oid_t id; + bson_buffer gte_buf; + bson gte_bson; + bson_buffer query_buf; + bson query_bson; + bson_buffer orderby_buf; + bson orderby_bson; + bson_buffer command_buf; + bson command_bson; + + bson_find(&it, gfile->meta, "_id"); + id = *bson_iterator_oid(&it); + + bson_buffer_init(&query_buf); + bson_append_oid(&query_buf, "files_id", &id); + if (size == 1) { + bson_append_int(&query_buf, "n", start); + } else { + bson_buffer_init(>e_buf); + bson_append_int(>e_buf, "$gte", start); + bson_from_buffer(>e_bson, >e_buf); + bson_append_bson(&query_buf, "n", >e_bson); + } + bson_from_buffer(&query_bson, &query_buf); + + bson_buffer_init(&orderby_buf); + bson_append_int(&orderby_buf, "n", 1); + bson_from_buffer(&orderby_bson, &orderby_buf); + + bson_buffer_init(&command_buf); + bson_append_bson(&command_buf, "query", &query_bson); + bson_append_bson(&command_buf, "orderby", &orderby_bson); + bson_from_buffer(&command_bson, &command_buf); + + return mongo_find(gfile->gfs->client, gfile->gfs->chunks_ns, + &command_bson, NULL, size, 0, 0); +} + +/*--------------------------------------------------------------------*/ + +gridfs_offset gridfile_write_file(gridfile* gfile, FILE *stream) + +{ + int i; + size_t len; + bson chunk; + bson_iterator it; + const char* data; + const int num = gridfile_get_numchunks( gfile ); + + for ( i=0; ipos < size) + ? contentlength - gfile->pos + : size; + bytes_left = size; + + first_chunk = (gfile->pos)/chunksize; + last_chunk = (gfile->pos+size-1)/chunksize; + total_chunks = last_chunk - first_chunk + 1; + chunks = gridfile_get_chunks(gfile, first_chunk, total_chunks); + + for (i = 0; i < total_chunks; i++) { + mongo_cursor_next(chunks); + chunk = chunks->current; + bson_find(&it, &chunk, "data"); + chunk_len = bson_iterator_bin_len( &it ); + chunk_data = bson_iterator_bin_data( &it ); + if (i == 0) { + chunk_data += (gfile->pos)%chunksize; + chunk_len -= (gfile->pos)%chunksize; + } + if (bytes_left > chunk_len) { + memcpy(buf, chunk_data, chunk_len); + bytes_left -= chunk_len; + buf += chunk_len; + } else { + memcpy(buf, chunk_data, bytes_left); + } + } + + mongo_cursor_destroy(chunks); + gfile->pos = gfile->pos + size; + + return size; +} + +/*--------------------------------------------------------------------*/ + +gridfs_offset gridfile_seek(gridfile* gfile, gridfs_offset offset) + +{ + gridfs_offset length; + + length = gridfile_get_contentlength(gfile); + gfile->pos = length < offset ? length : offset; + return gfile->pos; +} diff --git a/src/gridfs.h b/src/gridfs.h new file mode 100644 index 0000000..f0b4114 --- /dev/null +++ b/src/gridfs.h @@ -0,0 +1,278 @@ +/*--------------------------------------------------------------------*/ +/* gridfs.h */ +/* Author: Christopher Triolo */ +/*--------------------------------------------------------------------*/ + +#include "mongo.h" +#include "bson.h" +#include "platform_hacks.h" +#include +#ifndef GRIDFS_INCLUDED +#define GRIDFS_INCLUDED + +enum {DEFAULT_CHUNK_SIZE = 256 * 1024}; + +typedef uint64_t gridfs_offset; + +/* A GridFS contains a db connection, a root database name, and an + optional prefix */ +typedef struct { + /* The client to db-connection. */ + mongo_connection* client; + /* The root database name */ + const char* dbname; + /* The prefix of the GridFS's collections, default is NULL */ + const char* prefix; + /* The namespace where the file's metadata is stored */ + const char* files_ns; + /* The namespace where the files's data is stored in chunks */ + const char* chunks_ns; + +} gridfs; + +/* The state of a gridfile. This is used for incrementally writing buffers + * to a single GridFS file. + */ + +/* A GridFile contains the GridFS it is located in and the file + metadata */ +typedef struct { + /* The GridFS where the GridFile is located */ + gridfs* gfs; + /* The GridFile's bson object where all its metadata is located */ + bson* meta; + /* The position is the offset in the file */ + gridfs_offset pos; + /* The files_id of the gridfile */ + bson_oid_t id; + /* The name of the gridfile as a string */ + const char* remote_name; + /* The gridfile's content type */ + const char* content_type; + /* The length of this gridfile */ + gridfs_offset length; + /* The number of the current chunk being written to */ + int chunk_num; + /* A buffer storing data still to be written to chunks */ + char* pending_data; + /* Length of pending data */ + int pending_len; + +} gridfile; + +/*--------------------------------------------------------------------*/ + +/** Initializes a GridFS object + * @param client - db connection + * @param dbname - database name + * @param prefix - collection prefix, default is fs if NULL or empty + * @param gfs - the GridFS object to initialize + * @return - 1 if successful, 0 otherwise + */ +int gridfs_init(mongo_connection* client, const char* dbname, + const char* prefix, gridfs* gfs); + +/** Destroys a GridFS object + */ +void gridfs_destroy( gridfs* gfs ); + +/** Initializes a gridfile for writing incrementally with gridfs_write_buffer. + * Once initialized, you can write any number of buffers with gridfs_write_buffer. + * When done, you must call gridfs_writer_done to save the file metadata. + * + * @return - 1 if successful, 0 otherwise + */ +void gridfile_writer_init( gridfile* gfile, gridfs* gfs, const char* remote_name, const char* content_type ); + +/** Write to a GridFS file incrementally. You can call this function any number + * of times with a new buffer each time. This allows you to effectively + * stream to a GridFS file. When finished, be sure to call gridfs_writer_done. + * + * @return - 1 if successful, 0 otherwise + */ +void gridfile_write_buffer( gridfile* gfile, const char* data, gridfs_offset length ); + +/** Signal that writing of this gridfile is complete by + * writing any buffered chunks along with the entry in the + * files collection. + * + * @return - the file object if successful; otherwise 0. + */ +bson gridfile_writer_done( gridfile* gfile ); + +/** Store a buffer as a GridFS file. + * @param gfs - the working GridFS + * @param data - pointer to buffer to store in GridFS + * @param length - length of the buffer + * @param remotename - filename for use in the database + * @param contenttype - optional MIME type for this object + * @return - the file object + */ +bson gridfs_store_buffer(gridfs* gfs, const char* data, gridfs_offset length, + const char* remotename, + const char * contenttype); + +/** Open the file referenced by filename and store it as a GridFS file. + * @param gfs - the working GridFS + * @param filename - local filename relative to the process + * @param remotename - optional filename for use in the database + * @param contenttype - optional MIME type for this object + * @return - the file object + */ +bson gridfs_store_file(gridfs* gfs, const char* filename, + const char* remotename, const char* contenttype); + +/** Removes the files referenced by filename from the db + * @param gfs - the working GridFS + * @param filename - the filename of the file/s to be removed + */ +void gridfs_remove_filename(gridfs* gfs, const char* filename); + +/** Find the first query within the GridFS and return it as a GridFile + * @param gfs - the working GridFS + * @param query - a pointer to the bson with the query data + * @param gfile - the output GridFile to be initialized + * @return 1 if successful, 0 otherwise + */ +int gridfs_find_query(gridfs* gfs, bson* query, gridfile* gfile ); + +/** Find the first file referenced by filename within the GridFS + * and return it as a GridFile + * @param gfs - the working GridFS + * @param filename - filename of the file to find + * @param gfile - the output GridFile to be intialized + * @return 1 if successful, 0 otherwise + */ +int gridfs_find_filename(gridfs* gfs, const char *filename, + gridfile* gfile); + +/*--------------------------------------------------------------------*/ + + +/** Initializes a GridFile containing the GridFS and file bson + * @param gfs - the GridFS where the GridFile is located + * @param meta - the file object + * @param gfile - the output GridFile that is being initialized + * @return 1 if successful, 0 otherwise + */ +int gridfile_init(gridfs* gfs, bson* meta, gridfile* gfile); + +/** Destroys the GridFile + * @param oGridFIle - the GridFile being destroyed + */ +void gridfile_destroy(gridfile* gfile); + +/** Returns whether or not the GridFile exists + * @param gfile - the GridFile being examined + */ +int gridfile_exists(gridfile* gfile); + +/** Returns the filename of GridFile + * @param gfile - the working GridFile + * @return - the filename of the Gridfile + */ +const char * gridfile_get_filename(gridfile* gfile); + +/** Returns the size of the chunks of the GridFile + * @param gfile - the working GridFile + * @return - the size of the chunks of the Gridfile + */ +int gridfile_get_chunksize(gridfile* gfile); + +/** Returns the length of GridFile's data + * @param gfile - the working GridFile + * @return - the length of the Gridfile's data + */ +gridfs_offset gridfile_get_contentlength(gridfile* gfile); + +/** Returns the MIME type of the GridFile + * @param gfile - the working GridFile + * @return - the MIME type of the Gridfile + * (NULL if no type specified) + */ +const char* gridfile_get_contenttype(gridfile* gfile); + +/** Returns the upload date of GridFile + * @param gfile - the working GridFile + * @return - the upload date of the Gridfile + */ +bson_date_t gridfile_get_uploaddate(gridfile* gfile); + +/** Returns the MD5 of GridFile + * @param gfile - the working GridFile + * @return - the MD5 of the Gridfile + */ +const char* gridfile_get_md5(gridfile* gfile); + +/** Returns the field in GridFile specified by name + * @param gfile - the working GridFile + * @param name - the name of the field to be returned + * @return - the data of the field specified + * (NULL if none exists) + */ +const char *gridfile_get_field(gridfile* gfile, + const char* name); + +/** Returns a boolean field in GridFile specified by name + * @param gfile - the working GridFile + * @param name - the name of the field to be returned + * @return - the boolean of the field specified + * (NULL if none exists) + */ +bson_bool_t gridfile_get_boolean(gridfile* gfile, + const char* name); + +/** Returns the metadata of GridFile + * @param gfile - the working GridFile + * @return - the metadata of the Gridfile in a bson object + * (an empty bson is returned if none exists) + */ +bson gridfile_get_metadata(gridfile* gfile); + +/** Returns the number of chunks in the GridFile + * @param gfile - the working GridFile + * @return - the number of chunks in the Gridfile + */ +int gridfile_get_numchunks(gridfile* gfile); + +/** Returns chunk n of GridFile + * @param gfile - the working GridFile + * @return - the nth chunk of the Gridfile + */ +bson gridfile_get_chunk(gridfile* gfile, int n); + +/** Returns a mongo_cursor of *size* chunks starting with chunk *start* + * @param gfile - the working GridFile + * @param start - the first chunk in the cursor + * @param size - the number of chunks to be returned + * @return - mongo_cursor of the chunks (must be destroyed after use) + */ +mongo_cursor* gridfile_get_chunks(gridfile* gfile, int start, int size); + +/** Writes the GridFile to a stream + * @param gfile - the working GridFile + * @param stream - the file stream to write to + */ +gridfs_offset gridfile_write_file(gridfile* gfile, FILE* stream); + +/** Reads length bytes from the GridFile to a buffer + * and updates the position in the file. + * (assumes the buffer is large enough) + * (if size is greater than EOF gridfile_read reads until EOF) + * @param gfile - the working GridFile + * @param size - the amount of bytes to be read + * @param buf - the buffer to read to + * @return - the number of bytes read + */ +gridfs_offset gridfile_read(gridfile* gfile, gridfs_offset size, char* buf); + +/** Updates the position in the file + * (If the offset goes beyond the contentlength, + * the position is updated to the end of the file.) + * @param gfile - the working GridFile + * @param offset - the position to update to + * @return - resulting offset location + */ +gridfs_offset gridfile_seek(gridfile* gfile, gridfs_offset offset); + +#endif diff --git a/src/mongo.c b/src/mongo.c index 76b7736..1ed414c 100644 --- a/src/mongo.c +++ b/src/mongo.c @@ -63,7 +63,7 @@ void mongo_message_send(mongo_connection * conn, mongo_message* mm){ bson_little_endian32(&head.id, &mm->head.id); bson_little_endian32(&head.responseTo, &mm->head.responseTo); bson_little_endian32(&head.op, &mm->head.op); - + MONGO_TRY{ looping_write(conn, &head, sizeof(head)); looping_write(conn, &mm->data, mm->head.len - sizeof(head)); @@ -115,17 +115,18 @@ static int mongo_connect_helper( mongo_connection * conn ){ memset( conn->sa.sin_zero , 0 , sizeof(conn->sa.sin_zero) ); conn->sa.sin_family = AF_INET; conn->sa.sin_port = htons(conn->left_opts->port); - conn->sa.sin_addr.s_addr = inet_addr( conn->left_opts->host ); + conn->sa.sin_addr.s_addr = inet_addr(conn->left_opts->host); conn->addressSize = sizeof(conn->sa); /* connect */ conn->sock = socket( AF_INET, SOCK_STREAM, 0 ); if ( conn->sock <= 0 ){ + mongo_close_socket( conn->sock ); return mongo_conn_no_socket; } - - if ( connect( conn->sock , (struct sockaddr*)&conn->sa , conn->addressSize ) ){ - printf("failed to connect %s %d\n", conn->left_opts->host, conn->left_opts->port); + int result = 0; + if ( result=connect( conn->sock , (struct sockaddr*)&conn->sa , conn->addressSize ) ){ + mongo_close_socket( conn->sock ); return mongo_conn_fail; } @@ -154,6 +155,94 @@ mongo_conn_return mongo_connect( mongo_connection * conn , mongo_connection_opti return mongo_connect_helper(conn); } + +void mongo_replset_init_conn(mongo_connection* conn) { + conn->seeds = NULL; +} + +int mongo_replset_add_seed(mongo_connection* conn, const char* host, int port) { + mongo_host_port* host_port = bson_malloc(sizeof(mongo_host_port)); + host_port->port = port; + host_port->next = NULL; + strncpy( host_port->host, host, strlen(host) ); + + if( conn->seeds == NULL ) + conn->seeds = host_port; + else { + mongo_host_port* p = conn->seeds; + while( p->next != NULL ) + p = p->next; + p->next = host_port; + } + + return 0; +} + +mongo_conn_return mongo_replset_connect(mongo_connection* conn) { + + bson* out; + bson_bool_t ismaster; + + mongo_host_port* node = conn->seeds; + + conn->sock = 0; + conn->connected = 0; + + while( node != NULL ) { + + memset( conn->sa.sin_zero , 0 , sizeof(conn->sa.sin_zero) ); + conn->sa.sin_family = AF_INET; + conn->sa.sin_port = htons(node->port); + conn->sa.sin_addr.s_addr = inet_addr(node->host); + + conn->addressSize = sizeof(conn->sa); + + conn->sock = socket( AF_INET, SOCK_STREAM, 0 ); + if ( conn->sock <= 0 ){ + mongo_close_socket( conn->sock ); + return mongo_conn_no_socket; + } + + if ( connect( conn->sock , (struct sockaddr*)&conn->sa , conn->addressSize ) ){ + mongo_close_socket( conn->sock ); + } + + setsockopt( conn->sock, IPPROTO_TCP, TCP_NODELAY, (char *) &one, sizeof(one) ); + + /* Check whether this is the primary node */ + ismaster = 0; + + out = bson_malloc(sizeof(bson)); + out->data = NULL; + out->owned = 0; + + if (mongo_simple_int_command(conn, "admin", "ismaster", 1, out)) { + bson_iterator it; + bson_find(&it, out, "ismaster"); + ismaster = bson_iterator_bool(&it); + free(out); + } + + if(ismaster) { + conn->connected = 1; + } + else { + mongo_close_socket( conn->sock ); + } + + node = node->next; + } + + /* TODO signals */ + + /* Might be nice to know which node is primary */ + /* con->primary = NULL; */ + if( conn->connected == 1 ) + return 0; + else + return -1; +} + static void swap_repl_pair(mongo_connection * conn){ mongo_connection_options * tmp = conn->left_opts; conn->left_opts = conn->right_opts; @@ -175,7 +264,7 @@ mongo_conn_return mongo_connect_pair( mongo_connection * conn , mongo_connection memcpy( conn->left_opts, left, sizeof( mongo_connection_options ) ); memcpy( conn->right_opts, right, sizeof( mongo_connection_options ) ); - + return mongo_reconnect(conn); } @@ -294,6 +383,10 @@ mongo_reply * mongo_read_response( mongo_connection * conn ){ looping_read(conn, &fields, sizeof(fields)); bson_little_endian32(&len, &head.len); + + if (len < sizeof(head)+sizeof(fields) || len > 64*1024*1024) + MONGO_THROW(MONGO_EXCEPT_NETWORK); /* most likely corruption */ + out = (mongo_reply*)bson_malloc(len); out->head.len = len; @@ -318,7 +411,7 @@ mongo_reply * mongo_read_response( mongo_connection * conn ){ mongo_cursor* mongo_find(mongo_connection* conn, const char* ns, bson* query, bson* fields, int nToReturn, int nToSkip, int options){ int sl; - mongo_cursor * cursor; + volatile mongo_cursor * cursor; /* volatile due to longjmp in mongo exception handler */ char * data; mongo_message * mm = mongo_message_create( 16 + /* header */ 4 + /* options */ @@ -331,13 +424,13 @@ mongo_cursor* mongo_find(mongo_connection* conn, const char* ns, bson* query, bs data = &mm->data; data = mongo_data_append32( data , &options ); - data = mongo_data_append( data , ns , strlen( ns ) + 1 ); + data = mongo_data_append( data , ns , strlen( ns ) + 1 ); data = mongo_data_append32( data , &nToSkip ); data = mongo_data_append32( data , &nToReturn ); - data = mongo_data_append( data , query->data , bson_size( query ) ); + data = mongo_data_append( data , query->data , bson_size( query ) ); if ( fields ) - data = mongo_data_append( data , fields->data , bson_size( fields ) ); - + data = mongo_data_append( data , fields->data , bson_size( fields ) ); + bson_fatal_msg( (data == ((char*)mm) + mm->head.len), "query building fail!" ); mongo_message_send( conn , mm ); @@ -347,7 +440,7 @@ mongo_cursor* mongo_find(mongo_connection* conn, const char* ns, bson* query, bs MONGO_TRY{ cursor->mm = mongo_read_response(conn); }MONGO_CATCH{ - free(cursor); + free((mongo_cursor*)cursor); /* cast away volatile, not changing type */ MONGO_RETHROW(); } @@ -355,13 +448,13 @@ mongo_cursor* mongo_find(mongo_connection* conn, const char* ns, bson* query, bs cursor->ns = bson_malloc(sl); if (!cursor->ns){ free(cursor->mm); - free(cursor); + free((mongo_cursor*)cursor); /* cast away volatile, not changing type */ return 0; } memcpy((void*)cursor->ns, ns, sl); /* cast needed to silence GCC warning */ cursor->conn = conn; cursor->current.data = NULL; - return cursor; + return (mongo_cursor*)cursor; } bson_bool_t mongo_find_one(mongo_connection* conn, const char* ns, bson* query, bson* fields, bson* out){ @@ -399,7 +492,7 @@ int64_t mongo_count(mongo_connection* conn, const char* db, const char* ns, bson bson_destroy(&cmd); MONGO_RETHROW(); } - + bson_destroy(&cmd); bson_destroy(&out); return count; @@ -409,15 +502,11 @@ bson_bool_t mongo_disconnect( mongo_connection * conn ){ if ( ! conn->connected ) return 1; -#ifdef _WIN32 - closesocket( conn->sock ); -#else - close( conn->sock ); -#endif - + mongo_close_socket( conn->sock ); + conn->sock = 0; conn->connected = 0; - + return 0; } @@ -503,7 +592,7 @@ void mongo_cursor_destroy(mongo_cursor* cursor){ data = mongo_data_append32(data, &zero); data = mongo_data_append32(data, &one); data = mongo_data_append64(data, &cursor->mm->fields.cursorID); - + MONGO_TRY{ mongo_message_send(conn, mm); }MONGO_CATCH{ @@ -513,7 +602,7 @@ void mongo_cursor_destroy(mongo_cursor* cursor){ MONGO_RETHROW(); } } - + free(cursor->mm); free((void*)cursor->ns); free(cursor); @@ -542,7 +631,7 @@ bson_bool_t mongo_create_index(mongo_connection * conn, const char * ns, bson * bson_append_bool(&bb, "unique", 1); if (options & MONGO_INDEX_DROP_DUPS) bson_append_bool(&bb, "dropDups", 1); - + bson_from_buffer(&b, &bb); strncpy(idxns, ns, 1024-16); @@ -580,6 +669,7 @@ bson_bool_t mongo_run_command(mongo_connection * conn, const char * db, bson * c free(ns); return success; } + bson_bool_t mongo_simple_int_command(mongo_connection * conn, const char * db, const char* cmdstr, int arg, bson * realout){ bson out; bson cmd; @@ -595,7 +685,7 @@ bson_bool_t mongo_simple_int_command(mongo_connection * conn, const char * db, c if(bson_find(&it, &out, "ok")) success = bson_iterator_bool(&it); } - + bson_destroy(&cmd); if (realout) @@ -621,7 +711,7 @@ bson_bool_t mongo_simple_str_command(mongo_connection * conn, const char * db, c if(bson_find(&it, &out, "ok")) success = bson_iterator_bool(&it); } - + bson_destroy(&cmd); if (realout) @@ -653,7 +743,7 @@ static bson_bool_t mongo_cmd_get_error_helper(mongo_connection * conn, const cha bson_iterator it; haserror = (bson_find(&it, &out, "err") != bson_null); } - + if(realout) *realout = out; /* transfer of ownership */ else @@ -714,7 +804,7 @@ void mongo_cmd_add_user(mongo_connection* conn, const char* db, const char* user bson user_obj; bson pass_obj; char hex_digest[33]; - char* ns = malloc(strlen(db) + strlen(".system.users") + 1); + char* ns = bson_malloc(strlen(db) + strlen(".system.users") + 1); strcpy(ns, db); strcpy(ns+strlen(db), ".system.users"); diff --git a/src/mongo.h b/src/mongo.h index 50c4375..6a8ccd7 100644 --- a/src/mongo.h +++ b/src/mongo.h @@ -24,6 +24,7 @@ #ifdef _WIN32 #include #include +#define mongo_close_socket(sock) ( closesocket(sock) ) typedef int socklen_t; #else #include @@ -31,6 +32,7 @@ typedef int socklen_t; #include #include #include +#define mongo_close_socket(sock) ( close(sock) ) #endif MONGO_EXTERN_C_START @@ -40,9 +42,16 @@ typedef struct mongo_connection_options { int port; } mongo_connection_options; +typedef struct mongo_host_port { + char host[255]; + int port; + struct mongo_host_port* next; +} mongo_host_port; + typedef struct { mongo_connection_options* left_opts; /* always current server */ mongo_connection_options* right_opts; /* unused with single server */ + mongo_host_port* seeds; struct sockaddr_in sa; socklen_t addressSize; int sock; @@ -113,10 +122,13 @@ typedef enum { mongo_conn_return mongo_connect( mongo_connection * conn , mongo_connection_options * options ); mongo_conn_return mongo_connect_pair( mongo_connection * conn , mongo_connection_options * left, mongo_connection_options * right ); mongo_conn_return mongo_reconnect( mongo_connection * conn ); /* you will need to reauthenticate after calling */ -bson_bool_t mongo_disconnect( mongo_connection * conn ); /* use this if you want to be able to reconnect */ -bson_bool_t mongo_destroy( mongo_connection * conn ); /* you must call this even if connection failed */ +void mongo_replset_init_conn(mongo_connection* conn); +int mongo_replset_add_seed(mongo_connection* conn, const char* host, int port); +mongo_conn_return mongo_replset_connect(mongo_connection* conn); +bson_bool_t mongo_disconnect( mongo_connection * conn ); /* use this if you want to be able to reconnect */ +bson_bool_t mongo_destroy( mongo_connection * conn ); /* you must call this even if connection failed */ /* ---------------------------- CORE METHODS - insert update remove query getmore diff --git a/src/platform_hacks.h b/src/platform_hacks.h index 82c9799..88f228d 100644 --- a/src/platform_hacks.h +++ b/src/platform_hacks.h @@ -41,8 +41,10 @@ #include #elif defined(MONGO_USE__INT64) typedef __int64 int64_t; +typedef unsigned __int64 uint64_t; #elif defined(MONGO_USE_LONG_LONG_INT) typedef long long int int64_t; +typedef unsigned long long int uint64_t; #else #error must have a 64bit int type #endif