Skip to content

Commit

Permalink
Add support for missing Protobuf wellknown types (#2396)
Browse files Browse the repository at this point in the history 
Co-authored-by: Hui <khi@88665a586167.ant.amazon.com>
Co-authored-by: Carles Arnal <carlesarnal92@gmail.com>
  • Loading branch information
@hhkkxxx133 @carlesarnal
3 people authored Apr 7, 2022
1 parent 4daeab9 commit eab9faf
Show file tree
Hide file tree
Showing 4 changed files with 133 additions and 41 deletions.
5 changes: 5 additions & 0 deletions ...ilities/src/main/java/io/apicurio/registry/utils/protobuf/schema/FileDescriptorUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,11 @@ public static FileDescriptor[] baseDependencies() {
//Support all the Protobuf WellKnownTypes
//and the protos from Google API, https://github.com/googleapis/googleapis
return new FileDescriptor[] {
ApiProto.getDescriptor().getFile(),
FieldMaskProto.getDescriptor().getFile(),
SourceContextProto.getDescriptor().getFile(),
StructProto.getDescriptor().getFile(),
TypeProto.getDescriptor().getFile(),
TimestampProto.getDescriptor().getFile(),
WrappersProto.getDescriptor().getFile(),
AnyProto.getDescriptor().getFile(),
Expand Down
39 changes: 32 additions & 7 deletions ...lities/src/main/java/io/apicurio/registry/utils/protobuf/schema/ProtobufSchemaLoader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
public class ProtobufSchemaLoader {

private static final String GOOGLE_API_PATH = "google/type/";
private static final String GOOGLE_WELLKNOWN_PATH = "google/protobuf/";
//Adding pre-built support for commonly used Google API Protos,
//https://github.com/googleapis/googleapis
//These files need to be manually loaded into the FileSystem
Expand All @@ -47,6 +48,18 @@ public class ProtobufSchemaLoader {
.add("expr.proto")
.add("quaternion.proto")
.build();
//Adding support for Protobuf well-known types under package google.protobuf that are not covered by Square
//https://developers.google.com/protocol-buffers/docs/reference/google.protobuf
//These files need to be manually loaded into the FileSystem
//as Square doesn't support them by default.
private final static Set<String> GOOGLE_WELLKNOWN_PROTOS =
ImmutableSet.<String>builder()
.add("api.proto")
.add("field_mask.proto")
.add("source_context.proto")
.add("struct.proto")
.add("type.proto")
.build();

private static FileSystem getFileSystem() throws IOException {
final FileSystem inMemoryFileSystem =
Expand All @@ -58,25 +71,37 @@ private static FileSystem getFileSystem() throws IOException {
.setSupportedFeatures(Feature.SYMBOLIC_LINKS)
.build());

final ClassLoader classLoader = ProtobufSchemaLoader.class.getClassLoader();

createDirectory(GOOGLE_API_PATH.split("/"), inMemoryFileSystem);
loadProtoFiles(inMemoryFileSystem, classLoader, GOOGLE_API_PROTOS, GOOGLE_API_PATH);

final ClassLoader classLoader = ProtobufSchemaLoader.class.getClassLoader();
for (String googleApiProto : GOOGLE_API_PROTOS) {
createDirectory(GOOGLE_WELLKNOWN_PATH.split("/"), inMemoryFileSystem);
loadProtoFiles(inMemoryFileSystem, classLoader, GOOGLE_WELLKNOWN_PROTOS, GOOGLE_WELLKNOWN_PATH);

return inMemoryFileSystem;
}

private static void loadProtoFiles(FileSystem inMemoryFileSystem, ClassLoader classLoader, Set<String> protos,
String protoPath)
throws IOException {
for (String proto : protos) {
//Loads the proto file resource files.
final InputStream inputStream = classLoader.getResourceAsStream(GOOGLE_API_PATH + googleApiProto);
final InputStream inputStream = classLoader.getResourceAsStream(protoPath + proto);
final String fileContents = CharStreams.toString(new InputStreamReader(inputStream, Charsets.UTF_8));
final Path googleApiProtoPath = inMemoryFileSystem.getPath("/", GOOGLE_API_PATH, googleApiProto);
Files.write(googleApiProtoPath, fileContents.getBytes());
final Path path = inMemoryFileSystem.getPath("/", protoPath, proto);
Files.write(path, fileContents.getBytes());
}
return inMemoryFileSystem;
}

private static String createDirectory(String[] dirs, FileSystem fileSystem) throws IOException {
String dirPath = "";
for (String dir: dirs) {
dirPath = dirPath + "/" + dir;
Path path = fileSystem.getPath(dirPath);
Files.createDirectory(path);
if (Files.notExists(path)) {
Files.createDirectory(path);
}
}

return dirPath;
Expand Down
85 changes: 58 additions & 27 deletions utils/protobuf-schema-utilities/src/test/proto/WellKnownTypesTestSyntax2.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,50 +2,81 @@ syntax = "proto2";

package io.apicurio.registry.utils.protobuf.schema.syntax2;

import "google/protobuf/timestamp.proto";
import "google/protobuf/wrappers.proto";
import "google/protobuf/any.proto";
import "google/protobuf/empty.proto";
import "google/protobuf/api.proto";
import "google/protobuf/duration.proto";
import "google/type/money.proto";
import "google/type/date.proto";
import "google/type/timeofday.proto";
import "google/protobuf/empty.proto";
import "google/protobuf/field_mask.proto";
import "google/protobuf/source_context.proto";
import "google/protobuf/struct.proto";
import "google/protobuf/timestamp.proto";
import "google/protobuf/type.proto";
import "google/protobuf/wrappers.proto";
import "google/type/calendar_period.proto";
import "google/type/color.proto";
import "google/type/latlng.proto";
import "google/type/fraction.proto";
import "google/type/date.proto";
import "google/type/dayofweek.proto";
import "google/type/phone_number.proto";
import "google/type/postal_address.proto";
import "google/type/localized_text.proto";
import "google/type/month.proto";
import "google/type/interval.proto";
import "google/type/expr.proto";
import "google/type/fraction.proto";
import "google/type/latlng.proto";
import "google/type/money.proto";
import "google/type/month.proto";
import "google/type/postal_address.proto";
import "google/type/quaternion.proto";
import "google/type/timeofday.proto";
import "google/type/interval.proto";
import "google/type/localized_text.proto";
import "google/type/phone_number.proto";

message WellKnownTypesSyntax3 {
required int32 a = 100;
optional google.protobuf.Timestamp f1 = 1;
required google.protobuf.StringValue f2 = 2;
optional google.protobuf.FloatValue f3 = 3;
required google.protobuf.Empty f4 = 4;
repeated string bg = 101;
required float floating = 102;

repeated google.protobuf.Any f18 = 18;
optional google.protobuf.BoolValue f24 = 24;
optional google.protobuf.BytesValue f25 = 25;
optional google.protobuf.DoubleValue f26 = 26;
required google.protobuf.Duration f5 = 5;
required google.type.Money f6 = 6;
required google.type.Date f7 = 7;
optional google.type.TimeOfDay f8 = 8;
required google.protobuf.Empty f4 = 4;
optional google.protobuf.FloatValue f3 = 3;
optional google.protobuf.Int32Value f27 = 27;
optional google.protobuf.Int64Value f28 = 28;
optional google.protobuf.ListValue f22 = 22;
optional google.protobuf.NullValue f32 = 32;
required google.protobuf.StringValue f2 = 2;
optional google.protobuf.Struct f33 = 33;
optional google.protobuf.Timestamp f1 = 1;
optional google.protobuf.UInt32Value f34 = 34;
optional google.protobuf.UInt64Value f35 = 35;
optional google.protobuf.Value f36 = 36;
optional google.protobuf.FieldMask f37 = 37;
optional google.protobuf.Api f38 = 38;
optional google.protobuf.SourceContext f39 = 39;
optional google.protobuf.Type f41 = 41;
optional google.protobuf.Enum f42 = 42;
optional google.protobuf.EnumValue f43 = 43;
optional google.protobuf.Field f44 = 44;
optional google.protobuf.Field.Cardinality f45 = 45;
optional google.protobuf.Field.Kind f46 = 46;
optional google.protobuf.Method f47 = 47;
optional google.protobuf.Mixin f48 = 48;
optional google.protobuf.Option f49 = 49;
optional google.protobuf.Syntax f50 = 50;

required google.type.CalendarPeriod f9 = 9;
optional google.type.Color f10 = 10;
required google.type.Date f7 = 7;
optional google.type.DayOfWeek f11 = 11;
required float floating = 102;
optional google.type.LatLng f12 = 12;
optional google.type.Expr f20 = 20;
required google.type.Fraction f13 = 13;
optional google.type.LatLng f12 = 12;
required google.type.Money f6 = 6;
required google.type.Month f14 = 14;
optional google.type.PhoneNumber f15 = 15;
optional google.type.PostalAddress f16 = 16;
repeated google.type.Quaternion f21 = 21;
optional google.type.TimeOfDay f8 = 8;
optional google.type.PhoneNumber f15 = 15;
optional google.type.LocalizedText f17 = 17;
repeated google.protobuf.Any f18 = 18;
repeated google.type.Interval f19 = 19;
optional google.type.Expr f20 = 20;
repeated string bg = 101;
repeated google.type.Quaternion f21 = 21;
}
45 changes: 38 additions & 7 deletions utils/protobuf-schema-utilities/src/test/proto/WellKnownTypesTestSyntax3.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,11 @@ syntax = "proto3";

package io.apicurio.registry.utils.protobuf.schema.syntax3;

import "google/protobuf/api.proto";
import "google/protobuf/field_mask.proto";
import "google/protobuf/source_context.proto";
import "google/protobuf/type.proto";
import "google/protobuf/struct.proto";
import "google/protobuf/timestamp.proto";
import "google/protobuf/wrappers.proto";
import "google/protobuf/any.proto";
Expand All @@ -25,27 +30,53 @@ import "google/type/quaternion.proto";

message WellKnownTypesSyntax3 {
int32 a = 100;
google.protobuf.Timestamp f1 = 1;
google.protobuf.StringValue f2 = 2;
google.protobuf.FloatValue f3 = 3;
google.protobuf.Empty f4 = 4;
string bg = 101;
float floating = 102;

google.protobuf.Any f18 = 18;
google.protobuf.BoolValue f24 = 24;
google.protobuf.BytesValue f25 = 25;
google.protobuf.DoubleValue f26 = 26;
google.protobuf.Duration f5 = 5;
google.protobuf.Empty f4 = 4;
google.protobuf.FloatValue f3 = 3;
google.protobuf.Int32Value f27 = 27;
google.protobuf.Int64Value f28 = 28;
google.protobuf.ListValue f22 = 22;
google.protobuf.NullValue f29 = 29;
google.protobuf.StringValue f2 = 2;
google.protobuf.Struct f33 = 33;
google.protobuf.Timestamp f1 = 1;
google.protobuf.UInt32Value f34 = 34;
google.protobuf.UInt64Value f35 = 35;
google.protobuf.Value f36 = 36;
google.protobuf.Api f37 = 37;
google.protobuf.FieldMask f38 = 38;
google.protobuf.SourceContext f39 = 39;
google.protobuf.Type f41 = 41;
google.protobuf.Enum f42 = 42;
google.protobuf.EnumValue f43 = 43;
google.protobuf.Field f44 = 44;
google.protobuf.Field.Cardinality f45 = 45;
google.protobuf.Field.Kind f46 = 46;
google.protobuf.Method f47 = 47;
google.protobuf.Mixin f48 = 48;
google.protobuf.Option f49 = 49;
google.protobuf.Syntax f50 = 50;

google.type.Money f6 = 6;
google.type.Date f7 = 7;
google.type.TimeOfDay f8 = 8;
google.type.CalendarPeriod f9 = 9;
google.type.Color f10 = 10;
google.type.DayOfWeek f11 = 11;
float floating = 102;
google.type.LatLng f12 = 12;
google.type.Fraction f13 = 13;
google.type.Month f14 = 14;
google.type.PhoneNumber f15 = 15;
google.type.PostalAddress f16 = 16;
google.type.LocalizedText f17 = 17;
google.protobuf.Any f18 = 18;
google.type.Interval f19 = 19;
google.type.Expr f20 = 20;
string bg = 101;
google.type.Quaternion f21 = 21;
}

0 comments on commit eab9faf

Please sign in to comment.