Changeset 568

Timestamp:

06/30/08 09:33:00 (3 months ago)

Author:

mahlon

Message:

• Add a Rake task for generating tags, using exhuberant ctags.
  
• Extract album art from mp3 files.
  
• Prune unwanted id3 tags from mp3s
  
• Make the default handler use the new ThingFish::Request API.
  
• Restructure ThingFish::Request's public interface to request bodies.
  
• Update the ApacheBench? patch (PUT and DELETE support) for Apache 2.2.9.
  
• "Rollback" file storage on the event of a metadata store error.
  

Files:

• thingfish/trunk (modified) (1 prop)
• thingfish/trunk/Rakefile (modified) (1 diff)
• thingfish/trunk/experiments/ab_extra_methods.patch (modified) (10 diffs)
• thingfish/trunk/lib/thingfish/daemon.rb (modified) (1 diff)
• thingfish/trunk/lib/thingfish/handler/default.rb (modified) (3 diffs)
• thingfish/trunk/lib/thingfish/request.rb (modified) (6 diffs)
• thingfish/trunk/plugins/thingfish-filter-mp3/lib/thingfish/filter/mp3.rb (modified) (6 diffs)
• thingfish/trunk/plugins/thingfish-filter-mp3/spec/data (added)
• thingfish/trunk/plugins/thingfish-filter-mp3/spec/data/APIC-1-image.mp3 (added)
• thingfish/trunk/plugins/thingfish-filter-mp3/spec/data/APIC-2-images.mp3 (added)
• thingfish/trunk/plugins/thingfish-filter-mp3/spec/data/PIC-1-image.mp3 (added)
• thingfish/trunk/plugins/thingfish-filter-mp3/spec/data/PIC-2-images.mp3 (added)
• thingfish/trunk/plugins/thingfish-filter-mp3/spec/thingfish/filter/mp3_spec.rb (modified) (3 diffs)
• thingfish/trunk/spec/thingfish/daemon_spec.rb (modified) (2 diffs)
• thingfish/trunk/spec/thingfish/handler/default_spec.rb (modified) (7 diffs)
• thingfish/trunk/spec/thingfish/multipartmimeparser_spec.rb (modified) (1 diff)
• thingfish/trunk/spec/thingfish/request_spec.rb (modified) (1 diff)

thingfish/trunk/Rakefile

@@ -137 +137 @@
-
-
+### Task: generate ctags
+### This assumes exuberant ctags, since ctags 'native' doesn't support ruby anyway.
+desc "Generate a ctags 'tags' file from ThingFish source"
+task :ctags do
+    run %w{ ctags -R lib plugins misc }
+end
+
+
-### Cruisecontrol task
-desc "Cruisecontrol build"

thingfish/trunk/experiments/ab_extra_methods.patch

@@ -1 +1 @@
-
-
-
-
-
+
+
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-  * ab - or to due to a change in the distribution it is compiled with
-  * (such as an APR change in for example blocking).
-  */
-0.40-dev
-0.41-dev
+3
+3.1
- 
- /*
-  * BUGS:
-
-
+
- 
- int verbosity = 0;      /* no verbosity by default */
+ int recverrok = 0;      /* ok to proceed after socket receive errors */
--int posting = 0;        /* GET by default */
-/* GET by default
+ /* -2 => DELETE, -1 => HEAD, 0 => GET (default), 1 => POST, 2 => PUT
- int requests = 1;       /* Number of requests to make */
- int heartbeatres = 100; /* How often do we say we're alive */
- int concurrency = 1;    /* Number of multiple requests to make */
-21,7 +621
+31,7 +635
-             c->connect = tnow;
+             c->rwrote = 0;
-             c->rwrite = reqlen;
-             c->rwrote = 0;
--            if (posting)
-+            if (method)
@@ -35 +44 @@
-         }
-         else if (tnow > c->connect + aprtimeout) {
-61
+57
-     if (keepalive)
-l
-ld
+
+" APR_INT64_T_FMT "
--    if (posting > 0)
-+    if (method == 1)
-ld
+" APR_INT64_T_FMT "
-+    if (method == 2)
-ld
-ld
+" APR_INT64_T_FMT "
+" APR_INT64_T_FMT "
- 
-     /* avoid divide by zero */
-5
-(float) (1000 * timetaken / done)
+1
+    (double) timetaken * 1000 / done
-         printf("Transfer rate:          %.2f [Kbytes/sec] received\n",
-(float) (totalread / 1024 / timetaken)
+    (double) totalread / 1024 / timetaken
--        if (posting > 0) {
-+        if (method > 0) {
-             printf("                        %.2f kb/s sent\n",
-float) (totalposted / timetaken / 1024)
+double) totalposted / timetaken / 1024
-             printf("                        %.2f kb/s total\n",
-30,10 +1032
+42,10 +1048
-     printf("<tr %s><th colspan=2 %s>Total transferred:</th>"
-ld
+" APR_INT64_T_FMT "
-        trstring, tdstring, tdstring, totalread);
--    if (posting > 0)
-+    if (method == 1)
-         printf("<tr %s><th colspan=2 %s>Total POSTed:</th>"
-ld
+" APR_INT64_T_FMT "
-            trstring, tdstring, tdstring, totalposted);
-+    if (method == 2)
-+        printf("<tr %s><th colspan=2 %s>Total PUT:</th>"
-ld
+" APR_INT64_T_FMT "
-+           trstring, tdstring, tdstring, totalposted);
-     printf("<tr %s><th colspan=2 %s>HTML transferred:</th>"
-ld
+" APR_INT64_T_FMT "
-        trstring, tdstring, tdstring, totalbread);
-46,7 +1052
+58,7 +1068
-         printf("<tr %s><th colspan=2 %s>Transfer rate:</th>"
-            "<td colspan=2 %s>%.2f kb/s received</td></tr>\n",
-float) (totalread)
+double) totalread
--        if (posting > 0) {
-+        if (method > 0) {
@@ -81 +90 @@
-                "<td colspan=2 %s>%.2f kb/s sent</td></tr>\n",
-                trstring, tdstring, tdstring,
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-     }
- 
@@ -96 +116 @@
-+    if (method <= 0) {
-+        switch (method) {
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
-         snprintf_res = apr_snprintf(request, sizeof(_request),
-             "%s %s HTTP/1.0\r\n"
@@ -119 +138 @@
-     else {
-+        switch (method) {
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
-         snprintf_res = apr_snprintf(request,  sizeof(_request),
--            "POST %s HTTP/1.0\r\n"
@@ -135 +153 @@
-             "%s"
-             "\r\n",
-           
+
-             (isproxy) ? fullurl : path,
-             keepalive ? "Connection: Keep-Alive\r\n" : "",
-             cookie, auth,
-577,9 +1606
+622,9 +1653
-     reqlen = strlen(request);
- 
@@ -151 +169 @@
-         if (!buff) {
-             fprintf(stderr, "error creating request buffer: out of memory\n");
-
-
+
-     fprintf(stderr, "    -t timelimit    Seconds to max. wait for responses\n");
-
-
-
+
+
+
+
+
+
-     fprintf(stderr, "    -v verbosity    How much troubleshooting info to print\n");
-     fprintf(stderr, "    -w              Print out results in HTML tables\n");
@@ -165 +185 @@
-     fprintf(stderr, "    -y attributes   String to insert as tr attributes\n");
-     fprintf(stderr, "    -z attributes   String to insert as td or th attributes\n");
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
- #endif
- 
-     apr_getopt_init(&opt, cntxt, argc, argv);
-T:p:v:
-T:p:v:kVhwix:y:z:C:H:P:A:g:X:de:Sq:
+b:T:p:v:r
+b:T:p:v:rkVhwix:y:z:C:H:P:A:g:X:de:Sq
- #ifdef USE_SSL
-             "Z:f:"
- #endif
-1984,9 +2015
-concurrency
+2041,9 +2074
+windowsize
-                 break;
-             case 'i':
@@ -190 +250 @@
-             case 'g':
-                 gnuplot = strdup(optarg);
-01,10 +2035
+58,10 +2094
-                 confidence = 0;
-                 break;
@@ -214 +274 @@
-                 else if (postdata) {
-                     exit(r);
-

thingfish/trunk/lib/thingfish/daemon.rb

@@ -214 +214 @@
-
-        # Store some default metadata about the resource
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-        end
-        

thingfish/trunk/lib/thingfish/handler/default.rb

@@ -243 +243 @@
-    def handle_create_request( request, response )
-        
-entity_
-entity_
+
+
-            raise ThingFish::NotImplementedError, "multipart upload not implemented"
-        end
@@ -251 +251 @@
-
-        # Store the primary resource
-entity_
+
-        uuid = self.daemon.store_resource( body, metadata )
-    
@@ -271 +271 @@
-    def handle_update_uuid_request( request, response, uuid )
-        
+        if request.bodies.length > 1
+            self.log.error "Can't handle multipart request" % [ request.bodies ]
+            raise ThingFish::NotImplementedError, "multipart upload not implemented"
+        end
+
-        # :TODO: Handle slow/big uploads by returning '202 Accepted' and spawning
-        #         a handler thread?
-        new_resource = ! @filestore.has_file?( uuid )
-get_body_and_metadata
+bodies.to_a.flatten
-        self.daemon.store_resource( body, metadata, uuid )
-        

thingfish/trunk/lib/thingfish/request.rb

@@ -261 +261 @@
-
-
-
-
+
+
+
-    ### 
-body
-
-body
+resource
+resource 
+resource
-    ###    The new resource body as an IO-like object
-    ### +related_metadata+::
@@ -306 +307 @@
-
-    
-    ### Returns the entity bodies of the request along with any related metadata as
-    ### a Hash:
-    ### {
-    ###    <body io> => { <body metadata> },
-    ###    ...
-    ### }
-    def entity_bodies
-        # Parse the request's body parts if they aren't already
-        unless @entity_bodies
-            if self.has_multipart_body?
-                self.log.debug "Parsing multiple entity bodies."
-                @entity_bodies, @form_metadata = self.parse_multipart_body
-            else
-                self.log.debug "Parsing single entity body."
-                body, metadata = self.get_body_and_metadata
-                
-                @entity_bodies = { body => metadata }
-                @form_metadata = {}
-            end
-
-            self.log.debug "Parsed %d bodies and %d form_metadata (%p)" % 
-                [@entity_bodies.length, @form_metadata.length, @form_metadata.keys]
-        end
-
-        return @entity_bodies
-    end
-
-
-    ### Call the provided block once for each entity body of the request, which may
-_resources
+
-    ### is +true+, any resources which have been appended will be yielded immediately after the
-    ### body to which they are related. Note that this applies even in the current loop -- the
@@ -360 +333 @@
-    
-    
+    ### Fetch a flat hash of the entity bodies for the request. If +include_appended+ is +true+, 
+    ### include any appended related resources.
+    def bodies( include_appended=false )
+        rval = {}
+
+        self.each_body( include_appended ) do |body, metadata|
+            rval[body] = metadata
+        end
+
+        return rval
+    end
+    
+    
-    ### Check the body IO objects to ensure they're still open.
-    def check_body_ios
@@ -377 +363 @@
-                @entity_bodies[ clone ] = @entity_bodies.delete( body ) if @entity_bodies.key?( body )
-                @related_resources[ clone ] = @related_resources.delete( body ) if @related_resources.key?( body )
+                @metadata[ clone ] = @metadata.delete( body ) if @metadata.key?( body )
-                @related_resources.each do |_,hash|
-                    hash[ clone ] = hash.delete( body ) if hash.key?( body )
@@ -499 +486 @@
-    #########
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-        raise ArgumentError, "Can't return a single body for a multipart request" if
-            self.has_multipart_body?
-        
-default_
+
-            :useragent     => self.headers[ :user_agent ],
-
+
+
+
-        }
-
@@ -512 +529 @@
-        if self.headers[:content_disposition] &&
-            self.headers[:content_disposition] =~ /filename="(?:.*\\)?(.+?)"/i
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
-    end
-    

thingfish/trunk/plugins/thingfish-filter-mp3/lib/thingfish/filter/mp3.rb

@@ -24 +24 @@
-# Please see the LICENSE file for licensing details.
-#
+
-
-begin
@@ -56 +57 @@
-    NULL = "\x0"
-    
+    # Attached picture   "PIC"
+    #    Frame size         $xx xx xx
+    #    ---- mp3info is 'helpfully' cropping out frame size.
+    #    Text encoding      $xx
+    #    Image format       $xx xx xx
+    #    Picture type       $xx
+    #    Description        <textstring> $00 (00)
+    #    Picture data       <binary data>
+    PIC_FORMAT = 'h a3 h Z* a*'
+
+    # Attached picture "APIC"
+    #    Text encoding   $xx
+    #    MIME type       <text string> $00
+    #    Picture type    $xx
+    #    Description     <text string according to encoding> $00 (00)
+    #    Picture data    <binary data>
+    APIC_FORMAT = 'h Z* h Z* a*'
+    
-    # The Array of types this filter is interested in
-    HANDLED_TYPES = [ ThingFish::AcceptParam.parse('audio/mpeg') ]
@@ -83 +102 @@
-        request.each_body do |body, metadata|
-            if self.accept?( metadata[:format] )
-
+
+
+
+
+
+
+
+
+
-                request.append_metadata_for( body, mp3_metadata )
-                self.log.debug "Extracted mp3 info: %p" % [ mp3_metadata.keys ]
@@ -132 +159 @@
-    #########
-
-
-
-
-
+
+
-        mp3_metadata = {
-nfo
-nfo
-nfo
-nfo
-nfo
-nfo
-nfo
-nfo
-nfo
-nfo
+d3
+d3
+d3
+d3
+d3
+d3
+d3
+d3
+d3
+d3
-        }
-
-        # ID3V2 2.2.0 has three-letter tags, so map those if the artist info isn't set
-
-
+
+
+
-            
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
-            end
-        end
@@ -174 +193 @@
-        return sanitize_values( mp3_metadata )
-    end
+
+
+    ### Extract image data from id3 information, supports both APIC (2.3) and the older style
+    ### PIC (2.2).  Return value is a hash with IO keys and mimetype values.
+    ### {
+    ###    io  => { format => 'image/jpeg' }
+    ###    io2 => { format => 'image/jpeg' }
+    ### }
+    def extract_images( id3 )
+        data = {}
+        return data unless id3.hastag2?
+        
+        if id3.tag2.APIC
+            images = [ id3.tag2.APIC ].flatten
+            images.each do |img|
+                blob, mime = img.unpack( APIC_FORMAT ).values_at( 4, 1 )
+                data[ StringIO.new(blob) ] = { 
+                    :format => mime,
+                    :extent => blob.length,
+                    :title  => 'album-art'
+                }
+            end
+            
+        elsif id3.tag2.PIC
+            images = [ id3.tag2.PIC ].flatten
+            images.each do |img|
+                blob, type = img.unpack( PIC_FORMAT ).values_at( 4, 1 )
+                mime = MIMETYPE_MAP[ ".#{type.downcase}" ] or next
+                data[ StringIO.new(blob) ] = {
+                    :format => mime,
+                    :extent => blob.length,
+                    :title  => 'album-art'
+                }
+            end
+        end
+        
+        return data
+    end
+
-    
-    
@@ -199 +257 @@
-    end
-
-    
-    
-end # class ThingFish::Mp3Filter
-
-# vim: set nosta noet ts=4 sw=4:
-
-

thingfish/trunk/plugins/thingfish-filter-mp3/spec/thingfish/filter/mp3_spec.rb

@@ -54 +54 @@
-    }
-    
+    MP3_SPECDIR = Pathname.new( __FILE__ ).dirname.parent.parent
+    MP3_DATADIR = MP3_SPECDIR + 'data'
+
+    JPEG_MAGIC_REGEXP = /^\377\330\377\340\000\020JFIF/
+    PNG_MAGIC_REGEXP = /^\x89PNG/
+
-
-    before( :each ) do
-        @filter = ThingFish::Filter.create( 'mp3' )
-    
-        @io = StringIO.new( TEST_CONTENT )
-        @io.stub!( :path ).and_return( :a_dummy_path )
-        @response = stub( "response object" )
-
-        @request_metadata = { :format => 'audio/mpeg' }
-        @request = mock( "request object" )
-        @request.stub!( :http_method ).and_return( 'POST' )
-        @request.stub!( :each_body ).and_yield( @io, @request_metadata )
-
-        @mp3info = mock( "MP3 info object", :null_object => true )
-        Mp3Info.stub!( :new ).and_return( @mp3info )
-        @id3tag = mock( "ID3 tag object", :null_object => true )
-        @mp3info.stub!( :tag ).and_return( @id3tag )
-    end
-    
@@ -76 +72 @@
-    ### Shared behaviors
-    it_should_behave_like "A Filter"
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
-    it "ignores non-POST requests" do
-        @request.should_receive( :http_method ).
@@ -179 +81 @@
-        @filter.handle_request( @request, @response )
-    end
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+