Changeset 233 for trunk/ext

Timestamp:

05/08/2006 23:03:50 (3 years ago)

Author:

why

Message:

• ext/python: removal of the Python extension. Please use PySyck?!
  
• ext/ruby/ext/syck/rubyext.c: all patches from #1 (oceaN!), using to_f on incoming floats (strtod/rb_float_new isn't accurate).
  
• ext/ruby/lib/yaml/rubytypes.rb: added anonymous Class and Module loading/dumping. Thanks, Joel!
  
• ext/ruby/lib/yaml.rb: spelling error. Caught by ocean!
  
• lib/bytecode.c: re2c 0.10.0 is generating now.
  
• lib/token.rb: support loading of [a: 1, b: 2] inline syntax.
  
• lib/emitter.c: output of complex keys.
  
• lib/syck.c: fix from ocean (related to buffer overrun.)
  
• lib/syck.h: no longer need the definition of sycklex.
  

Location:

trunk/ext

Files:

• python (deleted)
• ruby/ext/syck/rubyext.c (modified) (44 diffs)
• ruby/lib/yaml.rb (modified) (1 diff)
• ruby/lib/yaml/rubytypes.rb (modified) (3 diffs)

trunk/ext/ruby/ext/syck/rubyext.c

@@ -57 +57 @@
  * symbols and constants
  */
-static ID s_new, s_utc, s_at, s_to_f, s_to_i, s_read, s_binmode, s_call, s_cmp, s_transfer, s_update, s_dup, s_haskey, s_match, s_keys, s_unpack, s_tr_bang, s_default_set, s_tag_read_class, s_tag_subclasses, s_resolver, s_push, s_emitter, s_level, s_detect_implicit, s_node_import, s_out, s_input, s_intern, s_transform, s_yaml_new, s_yaml_initialize, s_node_export, s_to_yaml, s_write, s_set_resolver;
-static ID s_tags, s_domain, s_kind, s_name, s_options, s_type_id, s_type_id_set, s_style, s_style_set, s_value, s_value_set;
+static ID s_new, s_utc, s_at, s_to_f, s_to_i, s_read, s_binmode, s_call, s_cmp, s_transfer, s_update, s_dup, s_haskey, s_match, s_keys, s_unpack, s_tr_bang, s_default_set, s_tag_read_class, s_tag_subclasses, s_resolver, s_push, s_emitter, s_level, s_detect_implicit, s_node_import, s_out, s_input, s_intern, s_transform, s_yaml_new, s_yaml_initialize, s_node_export, s_to_yaml, s_write, s_set_resolver, s_each;
+static ID s_tags, s_kind, s_name, s_options, s_type_id, s_type_id_set, s_style, s_style_set, s_value, s_value_set;
 static VALUE sym_model, sym_generic, sym_input, sym_bytecode;
 static VALUE sym_scalar, sym_seq, sym_map;
@@ -83 +83 @@
 void rb_syck_output_handler _((SyckEmitter *, char *, long));
 void rb_syck_emitter_handler _((SyckEmitter *, st_data_t));
-int syck_parser_assign_io _((SyckParser *, VALUE));
+int syck_parser_assign_io _((SyckParser *, VALUE *));
 VALUE syck_scalar_alloc _((VALUE class));
 VALUE syck_seq_alloc _((VALUE class));
@@ -115 +115 @@
 
     SyckParser *parser = syck_new_parser();
-    taint = syck_parser_assign_io(parser, port);
+    taint = syck_parser_assign_io(parser, &port);
     syck_parser_handler( parser, syck_yaml2byte_handler );
     syck_parser_error_handler( parser, NULL );
@@ -123 +123 @@
     syck_lookup_sym( parser, oid, (char **)&sav );
 
-    ret = S_ALLOC_N( char, strlen( sav->buffer ) + 3 );
+    ret = S_ALLOCA_N( char, strlen( sav->buffer ) + 3 );
     ret[0] = '\0';
     strcat( ret, "D\n" );
@@ -172 +172 @@
  */
 int
-syck_parser_assign_io(parser, port)
+syck_parser_assign_io(parser, pport)
     SyckParser *parser;
-    VALUE port;
+    VALUE *pport;
 {
     int taint = Qtrue;
-    VALUE tmp;
+    VALUE tmp, port = *pport;
     if (!NIL_P(tmp = rb_check_string_type(port))) {
         taint = OBJ_TAINTED(port); /* original taintedness */
@@ -192 +192 @@
         rb_raise(rb_eTypeError, "instance of IO needed");
     }
+    *pport = port;
     return taint;
 }
@@ -273 +274 @@
     if ( len > ptr - str && *ptr == '.' )
     {
-        char *padded = syck_strndup( "000000", 6 );
+        char padded[] = "000000";
         char *end = ptr + 1;
         while ( isdigit( *end ) ) end++;
         MEMCPY(padded, ptr + 1, char, end - (ptr + 1));
         usec = strtol(padded, NULL, 10);
-        S_FREE(padded);
     }
     else
@@ -462 +462 @@
                 double f;
                 syck_str_blow_away_commas( n );
-                f = strtod( n->data.str->ptr, NULL );
-                obj = rb_float_new( f );
+                obj = rb_str_new( n->data.str->ptr, n->data.str->len );
+                obj = rb_funcall( obj, s_to_f, 0 );
             }
             else if ( strcmp( type_id, "timestamp#iso8601" ) == 0 )
@@ -579 +579 @@
                             tmp = rb_ary_reverse( tmp );
                             rb_ary_push( tmp, obj );
-                            rb_iterate( rb_each, tmp, syck_merge_i, dup );
+                            rb_block_call( tmp, s_each, 0, 0, syck_merge_i, dup );
                             obj = dup;
                             skip_aset = 1;
@@ -603 +603 @@
 }
 
+static void syck_node_mark( SyckNode *n );
+
 /*
  * {native mode} node handler
@@ -711 +713 @@
 }
 
+static int
+syck_st_mark_nodes( char *key, SyckNode *n, char *arg )
+{
+    if ( n != (void *)1 ) syck_node_mark( n );
+    return ST_CONTINUE;
+}
+
 /*
  * mark parser nodes
@@ -718 +727 @@
     SyckParser *parser;
 {
-    struct parser_xtra *bonus;
-    rb_gc_mark(parser->root);
-    rb_gc_mark(parser->root_on_error);
-    if ( parser->bonus != NULL )
-    {
-        bonus = (struct parser_xtra *)parser->bonus;
+    struct parser_xtra *bonus = (struct parser_xtra *)parser->bonus;
+    rb_gc_mark_maybe(parser->root);
+    rb_gc_mark_maybe(parser->root_on_error);
         rb_gc_mark( bonus->data );
         rb_gc_mark( bonus->proc );
+    rb_gc_mark( bonus->resolver );
+
+    if ( parser->anchors != NULL )
+    {
+        st_foreach( parser->anchors, syck_st_mark_nodes, 0 );
+    }
+    if ( parser->bad_anchors != NULL )
+    {
+        st_foreach( parser->bad_anchors, syck_st_mark_nodes, 0 );
     }
 }
@@ -736 +751 @@
     SyckParser *p;
 {
-    struct parser_xtra *bonus = (struct parser_xtra *)p->bonus;
-    if ( bonus != NULL ) S_FREE( bonus );
+    S_FREE( p->bonus );
     syck_free_parser(p);
 }
@@ -751 +765 @@
     VALUE pobj;
     SyckParser *parser = syck_new_parser();
+
+    parser->bonus = S_ALLOC( struct parser_xtra );
+    S_MEMZERO( parser->bonus, struct parser_xtra, 1 );
 
     pobj = Data_Wrap_Struct( class, syck_mark_parser, rb_syck_free_parser, parser );
@@ -823 +840 @@
     VALUE port, proc, model, input;
     SyckParser *parser;
-    struct parser_xtra *bonus = S_ALLOC_N( struct parser_xtra, 1 );
-    volatile VALUE hash;    /* protect from GC */
+    struct parser_xtra *bonus;
 
     rb_scan_args(argc, argv, "11", &port, &proc);
@@ -833 +849 @@
     syck_set_model( self, input, model );
 
-    bonus->taint = syck_parser_assign_io(parser, port);
-    bonus->data = hash = rb_hash_new();
+    bonus = (struct parser_xtra *)parser->bonus;
+    bonus->taint = syck_parser_assign_io(parser, &port);
+    bonus->data = rb_hash_new();
     bonus->resolver = rb_attr_get( self, s_resolver );
     if ( NIL_P( proc ) ) bonus->proc = 0;
     else                 bonus->proc = proc;
     
-    parser->bonus = (void *)bonus;
-
     return syck_parse( parser );
 }
@@ -855 +870 @@
     VALUE port, proc, v, input, model;
     SyckParser *parser;
-    struct parser_xtra *bonus = S_ALLOC_N( struct parser_xtra, 1 );
-    volatile VALUE hash;
+    struct parser_xtra *bonus;
 
     rb_scan_args(argc, argv, "1&", &port, &proc);
@@ -865 +879 @@
     syck_set_model( self, input, model );
     
-    bonus->taint = syck_parser_assign_io(parser, port);
+    bonus = (struct parser_xtra *)parser->bonus;
+    bonus->taint = syck_parser_assign_io(parser, &port);
     bonus->resolver = rb_attr_get( self, s_resolver );
     bonus->proc = 0;
-    parser->bonus = (void *)bonus;
 
     while ( 1 )
     {
         /* Reset hash for tracking nodes */
-        bonus->data = hash = rb_hash_new();
+        bonus->data = rb_hash_new();
 
         /* Parse a document */
@@ -1000 +1014 @@
                             v = rb_ary_reverse( v );
                             rb_ary_push( v, obj );
-                            rb_iterate( rb_each, v, syck_merge_i, dup );
+                            rb_block_call( v, s_each, 0, 0, syck_merge_i, dup );
                             obj = dup;
                             skip_aset = 1;
@@ -1037 +1051 @@
     char *ivn;
     StringValue( ivname );
-    ivn = S_ALLOC_N( char, RSTRING(ivname)->len + 2 );
+    ivn = S_ALLOCA_N( char, RSTRING(ivname)->len + 2 );
     ivn[0] = '@';
     ivn[1] = '\0';
     strncat( ivn, RSTRING(ivname)->ptr, RSTRING(ivname)->len );
     rb_iv_set( obj, ivn, rb_ary_entry( vars, 1 ) );
-    S_FREE( ivn );
     return Qnil;
 }
@@ -1129 +1142 @@
                             subclass = cYObject;
                         }
+                        else /* workaround for SEGV. real fix please */
+                        {
+                            rb_raise( rb_eTypeError, "invalid subclass" );
+                        }
                     }
                     break;
@@ -1151 +1168 @@
             else if ( !NIL_P( target_class ) )
             {
+                if ( subclass == rb_cBignum )
+                {
+                    obj = rb_str2inum( val, 10 ); /* for yaml dumped by 1.8.3 [ruby-core:6159] */
+                }
+                else
+                {
                 obj = rb_obj_alloc( subclass );
+                }
+
                 if ( rb_respond_to( obj, s_yaml_initialize ) )
                 {
@@ -1158 +1183 @@
                 else if ( !NIL_P( obj ) && rb_obj_is_instance_of( val, rb_cHash ) )
                 {
-                    rb_iterate( rb_each, val, syck_set_ivars, obj );
+                    rb_block_call( val, s_each, 0, 0, syck_set_ivars, obj );
                 }
             }
@@ -1195 +1220 @@
     if ( !NIL_P(tmp) )
     {
-        char *taguri;
-        val = tmp;
-        taguri = syck_type_id_to_uri( RSTRING(val)->ptr );
-        return rb_str_new2( taguri );
+        char *taguri = syck_type_id_to_uri( RSTRING(tmp)->ptr );
+        val = rb_str_new2( taguri );
+        S_FREE( taguri );
     }
 
@@ -1388 +1412 @@
 {
     int i;
+    rb_gc_mark_maybe( n->id );
     switch ( n->kind )
     {
@@ -1405 +1430 @@
         break;
     }
-}
-
-/*
- * Don't free Ruby data, Ruby will do that
- */
-void
-rb_syck_free_node( SyckNode *n )
-{
-    switch ( n->kind  )
-    {
-        case syck_str_kind:
-            S_FREE( n->data.str );
-            n->data.str = NULL;
-        break;
-
-        case syck_seq_kind:
-            if ( n->data.list != NULL )
-            {
-                S_FREE( n->data.list->items );
-                S_FREE( n->data.list );
-                n->data.list = NULL;
-            }
-        break;
-
-        case syck_map_kind:
-            if ( n->data.pairs != NULL )
-            {
-                S_FREE( n->data.pairs->keys );
-                S_FREE( n->data.pairs->values );
-                S_FREE( n->data.pairs );
-                n->data.pairs = NULL;
-            }
-        break;
-    }
-
-    S_FREE( n );
+#if 0 /* maybe needed */
+    if ( n->shortcut ) syck_node_mark( n->shortcut ); /* caution: maybe cyclic */
+#endif
 }
 
@@ -1451 +1443 @@
 {
     SyckNode *node = syck_alloc_str();
-    VALUE obj = Data_Wrap_Struct( class, syck_node_mark, rb_syck_free_node, node );
+    VALUE obj = Data_Wrap_Struct( class, syck_node_mark, syck_free_node, node );
     node->id = obj;
     return obj;
@@ -1520 +1512 @@
 
     StringValue( val );
-    node->data.str->ptr = RSTRING(val)->ptr;
+    node->data.str->ptr = syck_strndup( RSTRING(val)->ptr, RSTRING(val)->len );
     node->data.str->len = RSTRING(val)->len;
     node->data.str->style = scalar_none;
@@ -1538 +1530 @@
     VALUE obj;
     node = syck_alloc_seq();
-    obj = Data_Wrap_Struct( class, syck_node_mark, rb_syck_free_node, node );
+    obj = Data_Wrap_Struct( class, syck_node_mark, syck_free_node, node );
     node->id = obj;
     return obj;
@@ -1637 +1629 @@
     VALUE obj;
     node = syck_alloc_map();
-    obj = Data_Wrap_Struct( class, syck_node_mark, rb_syck_free_node, node );
+    obj = Data_Wrap_Struct( class, syck_node_mark, syck_free_node, node );
     node->id = obj;
     return obj;
@@ -1767 +1759 @@
         return copy;
 
-    if ( TYPE( orig ) != T_DATA ||
-         RDATA( orig )->dfree != ( RUBY_DATA_FUNC )rb_syck_free_node )
+    if ( TYPE( orig ) != T_DATA )
     {
         rb_raise( rb_eTypeError, "wrong argument type" );
@@ -1789 +1780 @@
     Data_Get_Struct( self, SyckNode, node );
 
-    if ( node->type_id != NULL ) S_FREE( node->type_id );
-
-    if ( NIL_P( type_id ) ) {
-        node->type_id = NULL;
-    } else {
-        node->type_id = StringValuePtr( type_id );
+    S_FREE( node->type_id );
+
+    if ( !NIL_P( type_id ) ) {
+        StringValue( type_id );
+        node->type_id = syck_strndup( RSTRING(type_id)->ptr, RSTRING(type_id)->len );
     }
 
@@ -1812 +1802 @@
     SyckNode *orig_n;
     Data_Get_Struct(self, SyckNode, orig_n);
+    t = Data_Wrap_Struct( cNode, syck_node_mark, syck_free_node, 0 );
 
     switch (orig_n->kind)
@@ -1818 +1809 @@
             {
                 int i;
-                n = syck_alloc_map();
+                DATA_PTR(t) = n = syck_alloc_map();
                 for ( i = 0; i < orig_n->data.pairs->idx; i++ )
                 {
@@ -1830 +1821 @@
             {
                 int i;
-                n = syck_alloc_seq();
+                DATA_PTR(t) = n = syck_alloc_seq();
                 for ( i = 0; i < orig_n->data.list->idx; i++ )
                 {
@@ -1839 +1830 @@
 
         case syck_str_kind:
-            n = syck_new_str2( orig_n->data.str->ptr, orig_n->data.str->len, orig_n->data.str->style );
+            DATA_PTR(t) = n = syck_new_str2( orig_n->data.str->ptr, orig_n->data.str->len, orig_n->data.str->style );
         break;
     }
@@ -1851 +1842 @@
         n->anchor = syck_strndup( orig_n->anchor, strlen( orig_n->anchor ) );
     }
-    t = Data_Wrap_Struct( cNode, NULL, NULL, n );
     n->id = t;
-    t = rb_funcall( oDefaultResolver, s_node_import, 1, t );
-    syck_free_node( n );
-    return t;
+    return rb_funcall( oDefaultResolver, s_node_import, 1, t );
 }
 
@@ -1950 +1938 @@
     SyckEmitter *emitter;
 {
-    struct emitter_xtra *bonus;
-    if ( emitter->bonus != NULL )
-    {
-        bonus = (struct emitter_xtra *)emitter->bonus;
+    struct emitter_xtra *bonus = (struct emitter_xtra *)emitter->bonus;
+    rb_gc_mark( bonus->oid  );
         rb_gc_mark( bonus->data );
         rb_gc_mark( bonus->port );
     }
-}
 
 /*
@@ -1966 +1951 @@
     SyckEmitter *e;
 {
-    struct emitter_xtra *bonus = (struct emitter_xtra *)e->bonus;
-    if ( bonus != NULL ) S_FREE( bonus );
+    S_FREE( e->bonus );
     syck_free_emitter(e);
 }
@@ -1982 +1966 @@
     SyckEmitter *emitter = syck_new_emitter();
 
+    emitter->bonus = S_ALLOC( struct emitter_xtra );
+    S_MEMZERO( emitter->bonus, struct emitter_xtra, 1 );
+
     pobj = Data_Wrap_Struct( class, syck_mark_emitter, rb_syck_free_emitter, emitter );
     syck_emitter_handler( emitter, rb_syck_emitter_handler );
@@ -2002 +1989 @@
     SyckEmitter *emitter;
     struct emitter_xtra *bonus;
-    volatile VALUE hash;    /* protect from GC */
 
     Data_Get_Struct(self, SyckEmitter, emitter);
     bonus = (struct emitter_xtra *)emitter->bonus;
-    if ( bonus != NULL ) S_FREE( bonus );
-
-    bonus = S_ALLOC_N( struct emitter_xtra, 1 );
+
+    bonus->oid = Qnil;
     bonus->port = rb_str_new2( "" );
-    bonus->data = hash = rb_hash_new();
+    bonus->data = rb_hash_new();
 
     if (rb_scan_args(argc, argv, "01", &options) == 0)
@@ -2031 +2016 @@
     }
     
-    emitter->headless = 1;
-    emitter->bonus = (void *)bonus;
+    emitter->headless = 0;
     rb_ivar_set(self, s_level, INT2FIX(0));
     rb_ivar_set(self, s_resolver, Qnil);
@@ -2218 +2202 @@
     s_yaml_new = rb_intern("yaml_new");
     s_yaml_initialize = rb_intern("yaml_initialize");
+    s_each = rb_intern("each");
 
     s_tags = rb_intern("@tags");
@@ -2260 +2245 @@
     rb_define_method( cResolver, "tagurize", syck_resolver_tagurize, 1 );
 
+    rb_global_variable( &oDefaultResolver );
     oDefaultResolver = rb_funcall( cResolver, rb_intern( "new" ), 0 );
     rb_define_singleton_method( oDefaultResolver, "node_import", syck_defaultresolver_node_import, 1 );
     rb_define_singleton_method( oDefaultResolver, "detect_implicit", syck_defaultresolver_detect_implicit, 1 );
     rb_define_const( rb_syck, "DefaultResolver", oDefaultResolver );
+    rb_global_variable( &oGenericResolver );
     oGenericResolver = rb_funcall( cResolver, rb_intern( "new" ), 0 );
     rb_define_singleton_method( oGenericResolver, "node_import", syck_genericresolver_node_import, 1 );

trunk/ext/ruby/lib/yaml.rb

@@ -312 +312 @@
         # Add a transfer method for a builtin type
         #
-        def YAML.add_ruby_type( type, &transfer_proc )
+        def YAML.add_ruby_type( type_tag, &transfer_proc )
             resolver.add_type( "tag:ruby.yaml.org,2002:#{ type_tag }", transfer_proc )
         end

trunk/ext/ruby/lib/yaml/rubytypes.rb

@@ -3 +3 @@
 require 'yaml/compat'
 
+class Module
+    yaml_as "tag:ruby.yaml.org,2002:module"
+
+    def Module.yaml_new( klass, tag, val )
+        if String === val
+            val.split(/::/).inject(Object) do |m, n|
+                begin
+                    m.const_get(n)
+                rescue NameError
+                    raise ArgumentError, "undefined class/module #{n} in #{val}"
+                end
+            end
+        else
+            raise YAML::TypeError, "Invalid Module: " + val.inspect
+        end
+    end
+
+    def to_yaml( opts = {} )
+        YAML::quick_emit( nil, opts ) { |out|
+            out.scalar( "tag:ruby.yaml.org,2002:module", self.name, :plain )
+        }
+    end
+end
+
 class Class
-        def to_yaml( opts = {} )
-                raise TypeError, "can't dump anonymous class %s" % self.class
-        end
+    yaml_as "tag:ruby.yaml.org,2002:class"
+
+    def Class.yaml_new( klass, tag, val )
+            if String === val
+            val.split(/::/).inject(Object) do |m, n|
+                begin
+                    m.const_get(n)
+                rescue NameError
+                    raise ArgumentError, "undefined class/module #{n} in #{val}"
+                end
+            end
+        else
+            raise YAML::TypeError, "Invalid Class: " + val.inspect
+        end
+    end
+
+    def to_yaml( opts = {} )
+        YAML::quick_emit( nil, opts ) { |out|
+            out.scalar( "tag:ruby.yaml.org,2002:class", self.name, :plain )
+        }
+    end
 end
 
@@ -355 +397 @@
 end
 
-class Numeric
+class Integer
+    yaml_as "tag:yaml.org,2002:int"
+        def to_yaml( opts = {} )
+                YAML::quick_emit( nil, opts ) do |out|
+            out.scalar( "tag:yaml.org,2002:int", self.to_s, :plain )
+        end
+        end
+end
+
+class Float
+    yaml_as "tag:yaml.org,2002:float"
         def to_yaml( opts = {} )
                 YAML::quick_emit( nil, opts ) do |out|
@@ -366 +418 @@
                 str = ".NaN"
             end
-            out.scalar( taguri, str, :plain )
-        end
-        end
-end
-
-class Fixnum
-    yaml_as "tag:yaml.org,2002:int"
-end
-
-class Float
-    yaml_as "tag:yaml.org,2002:float"
+            out.scalar( "tag:yaml.org,2002:float", str, :plain )
+        end
+end
 end