Verwenden von Apache avro reflect

Question

Avro-Serialisierung ist bei Hadoop-Benutzern beliebt, aber Beispiele sind so schwer zu finden.

Kann mir jemand mit diesem Beispielcode helfen? Ich bin hauptsächlich daran interessiert, die Reflect-API zum Lesen/Schreiben in Dateien und zum Verwenden der Union- und Null-Anmerkungen zu verwenden.

public class Reflect { 

    public class Packet { 
     int cost; 
     @Nullable TimeStamp stamp; 
     public Packet(int cost, TimeStamp stamp){ 
      this.cost = cost; 
      this.stamp = stamp; 
     } 
    } 

    public class TimeStamp { 
     int hour = 0; 
     int second = 0; 
     public TimeStamp(int hour, int second){ 
      this.hour = hour; 
      this.second = second; 
     } 
    } 

    public static void main(String[] args) throws IOException { 
     TimeStamp stamp; 
     Packet packet; 

     stamp = new TimeStamp(12, 34); 
     packet = new Packet(9, stamp); 
     write(file, packet); 

     packet = new Packet(8, null); 
     write(file, packet); 
     file.close(); 

     // open file to read. 
     packet = read(file); 
     packet = read(file); 
    } 
} 

Answer 1

Hier ist eine Version des oben genannten Programms, das funktioniert.

Dies verwendet auch die Komprimierung für die Datei.

import java.io.File; 
import org.apache.avro.Schema; 
import org.apache.avro.file.DataFileWriter; 
import org.apache.avro.file.DataFileReader; 
import org.apache.avro.file.CodecFactory; 
import org.apache.avro.io.DatumWriter; 
import org.apache.avro.io.DatumReader; 
import org.apache.avro.reflect.ReflectData; 
import org.apache.avro.reflect.ReflectDatumWriter; 
import org.apache.avro.reflect.ReflectDatumReader; 
import org.apache.avro.reflect.Nullable; 

public class Reflect { 

    public static class Packet { 
    int cost; 
    @Nullable TimeStamp stamp; 
    public Packet() {}      // required to read 
    public Packet(int cost, TimeStamp stamp){ 
     this.cost = cost; 
     this.stamp = stamp; 
    } 
    } 

    public static class TimeStamp { 
    int hour = 0; 
    int second = 0; 
    public TimeStamp() {}      // required to read 
    public TimeStamp(int hour, int second){ 
     this.hour = hour; 
     this.second = second; 
    } 
    } 

    public static void main(String[] args) throws Exception { 
    // one argument: a file name 
    File file = new File(args[0]); 

    // get the reflected schema for packets 
    Schema schema = ReflectData.get().getSchema(Packet.class); 

    // create a file of packets 
    DatumWriter<Packet> writer = new ReflectDatumWriter<Packet>(Packet.class); 
    DataFileWriter<Packet> out = new DataFileWriter<Packet>(writer) 
     .setCodec(CodecFactory.deflateCodec(9)) 
     .create(schema, file); 

    // write 100 packets to the file, odds with null timestamp 
    for (int i = 0; i < 100; i++) { 
     out.append(new Packet(i, (i%2==0) ? new TimeStamp(12, i) : null)); 
    } 

    // close the output file 
    out.close(); 

    // open a file of packets 
    DatumReader<Packet> reader = new ReflectDatumReader<Packet>(Packet.class); 
    DataFileReader<Packet> in = new DataFileReader<Packet>(file, reader); 

    // read 100 packets from the file & print them as JSON 
    for (Packet packet : in) { 
     System.out.println(ReflectData.get().toString(packet)); 
    } 

    // close the input file 
    in.close(); 
    } 

} 

Answer 2

Siehe Beispiel 3 bei https://sites.google.com/site/developertips/Home/java/apache-avro Es zeigt, wie Sie das Reflection-API von Java-Klassen verwenden können, schreiben und zu lesen.