Changeset 9804

Timestamp:

10/30/10 00:34:48 (3 years ago)

Author:

SvenDowideit

Message:

Item9893: auto add indexes until it won't let you anymore - this isn't a solution, but I want to get it into the history so that later we know why we can't do this

Location:

trunk/MongoDBPlugin/lib/Foswiki/Plugins

Files:

• MongoDBPlugin.pm (modified) (1 diff)
• MongoDBPlugin/DB.pm (modified) (2 diffs)

trunk/MongoDBPlugin/lib/Foswiki/Plugins/MongoDBPlugin.pm

@@ -173 +173 @@
             my $FIELD = $savedMeta->{$key};
             $meta->{$key} = {};
+            
             foreach my $elem (@$FIELD) {
+                if ($key eq 'FIELD') {
+                    #TODO: move this into the search algo, so it makes an index the first time someone builds an app that sorts on it.
+                    #even then, we have a hard limit of 40 indexes, so we're going to have to get more creative.
+                    #mind you, we don't really need indexes for speed, just to cope with query() resultsets that contain more than 1Meg of documents - so maybe we can delay creation until that happens?
+                    getMongoDB()->ensureIndex( 'current', { $key.'.'.$elem->{name} => 1 }, {name=>$key.'.'.$elem->{name}});
+                }
+                
                 $meta->{$key}{ $elem->{name} } = $elem;
             }

trunk/MongoDBPlugin/lib/Foswiki/Plugins/MongoDBPlugin/DB.pm

@@ -49 +49 @@
 
 #TODO: not the most efficient place to create and index, but I want to be sure, to be sure.
-    $collection->ensure_index( { _topic => 1 } );
-    $collection->ensure_index( { _topic => 1, _web => 1 }, { unique => 1 } );
-#TODO: really should use the auto indexed '_id' (or maybe we can use this as a tuid - unique foreach rev of each topic..)
+    $self->ensureIndex( $collection, { _topic => 1 }, {name=>'_topic'});
+    $self->ensureIndex( $collection, { _topic => 1, _web => 1 }, { name=>'_topic:_web', unique => 1 } );
+    $self->ensureIndex( $collection, { 'TOPICINFO.author' => 1 }, {name=>'TOPICINFO.author'} );
+    $self->ensureIndex( $collection, { 'TOPICINFO.date' => 1  }, {name=>'TOPICINFO.date'} );
+    $self->ensureIndex( $collection, { 'TOPICPARENT.name' => 1  }, {name=>'TOPICPARENT.name'} );
+
+#TODO: maybe should use the auto indexed '_id' (or maybe we can use this as a tuid - unique foreach rev of each topic..)
+#then again, atm, its totally random, so may be good for sharding.
 
     $collection->update(
@@ -59 +64 @@
     );
 }
+
+
+#BUGGER. compound indexes won't help with large queries
+#> db.current.dropIndexes();                                     
+#{
+#       "nIndexesWas" : 2,
+#       "msg" : "non-_id indexes dropped for collection",
+#       "ok" : 1
+#}
+#> db.current.find().sort({_topic:1})
+#error: {
+#       "$err" : "too much data for sort() with no index.  add an index or specify a smaller limit",
+#       "code" : 10128
+#}
+#> db.current.ensureIndex({_web:1, _topic:1, 'TOPICINFO.date':1, 'TOPICINFO.author': 1});
+#> db.current.find().sort({_topic:1})
+#error: {
+#       "$err" : "too much data for sort() with no index.  add an index or specify a smaller limit",
+#       "code" : 10128
+#}
+#> 
+#    $collection->ensure_index( { 'TOPICINFO.author' => 1, 'TOPICINFO.date' => 1, 'TOPICPARENT.name' => 1  } );
+#    $collection->ensure_index( { 'TOPICINFO.author' => -1, 'TOPICINFO.date' => -1, 'TOPICPARENT.name' => -1  } );
+#MongoDB's ensure_index causes the server to re0index, even if that index already exists, so we need to wrap it.
+sub ensureIndex {
+    my $self = shift;
+    my $collection = shift; #either a collection object of a name
+    my $indexRef = shift;   #can be a hashref or an ixHash
+    my $options = shift;
+    
+    ASSERT(defined($options->{name})) if DEBUG;
+    
+    if (ref($collection) eq '') {
+        #convert name of collection to collection obj
+        $collection = $self->_getCollection($collection);
+    }
+    
+    #cache the indexes we know about
+    if (not defined($self->{mongoDBIndexes})) {
+        my @indexes = $collection->get_indexes();
+        $self->{mongoDBIndexes} = \@indexes;
+    }
+    foreach my $index (@{$self->{mongoDBIndexes}})  {
+        #print STDERR "we already have:  ".$index->{name}." index\n";
+        if ($options->{name} eq $index->{name}) {
+            #already exists, do nothing.
+            return;
+        }
+    }
+    if (scalar(@{$self->{mongoDBIndexes}}) >= 40) {
+        print STDERR "*******************ouch. MongoDB can only have 40 indexes per collection\n";
+        return;
+    }
+print STDERR "creating ".$options->{name}." index\n";
+    #TODO: consider doing these in a batch at the end of a request, or?
+    $collection->ensure_index($indexRef, $options);
+    undef $self->{mongoDBIndexes}; #clear the cache :/
+}
+
 
 sub remove {