/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.parquet.avro;

import static org.apache.avro.Schema.Type.INT;
import static org.apache.avro.Schema.Type.LONG;
import static org.apache.avro.Schema.Type.STRING;
import static org.apache.avro.SchemaCompatibility.SchemaCompatibilityType.COMPATIBLE;
import static org.apache.avro.SchemaCompatibility.checkReaderWriterCompatibility;
import static org.apache.parquet.avro.AvroTestUtil.array;
import static org.apache.parquet.avro.AvroTestUtil.field;
import static org.apache.parquet.avro.AvroTestUtil.optionalField;
import static org.apache.parquet.avro.AvroTestUtil.primitive;
import static org.apache.parquet.avro.AvroTestUtil.record;
import static org.apache.parquet.avro.AvroWriteSupport.WRITE_FIXED_AS_INT96;
import static org.apache.parquet.schema.OriginalType.DATE;
import static org.apache.parquet.schema.OriginalType.TIMESTAMP_MICROS;
import static org.apache.parquet.schema.OriginalType.TIMESTAMP_MILLIS;
import static org.apache.parquet.schema.OriginalType.TIME_MICROS;
import static org.apache.parquet.schema.OriginalType.TIME_MILLIS;
import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY;
import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BOOLEAN;
import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.DOUBLE;
import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY;
import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.FLOAT;
import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT32;
import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT64;
import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT96;
import static org.apache.parquet.schema.Type.Repetition.REQUIRED;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.Mockito.CALLS_REAL_METHODS;

import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.common.io.Resources;
import java.util.Arrays;
import java.util.Collections;
import org.apache.avro.JsonProperties;
import org.apache.avro.LogicalTypes;
import org.apache.avro.Schema;
import org.apache.hadoop.conf.Configuration;
import org.apache.parquet.schema.LogicalTypeAnnotation;
import org.apache.parquet.schema.MessageType;
import org.apache.parquet.schema.MessageTypeParser;
import org.apache.parquet.schema.PrimitiveType;
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
import org.apache.parquet.schema.Type;
import org.apache.parquet.schema.Types;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.MockedStatic;
import org.mockito.Mockito;

public class TestAvroSchemaConverter {

  private static final Configuration NEW_BEHAVIOR = new Configuration(false);
  private MockedStatic<AvroRecordConverter> avroRecordConverterMock;

  @BeforeEach
  public void setupMockito() {
    avroRecordConverterMock = Mockito.mockStatic(AvroRecordConverter.class, CALLS_REAL_METHODS);
  }

  @AfterEach
  public void tearDown() {
    avroRecordConverterMock.close();
  }

  @BeforeAll
  public static void setupConf() {
    NEW_BEHAVIOR.setBoolean("parquet.avro.add-list-element-records", false);
    NEW_BEHAVIOR.setBoolean("parquet.avro.write-old-list-structure", false);
  }

  public static final String ALL_PARQUET_SCHEMA = "message org.apache.parquet.avro.myrecord {\n"
      + "  required boolean myboolean;\n"
      + "  required int32 myint;\n"
      + "  required int64 mylong;\n"
      + "  required float myfloat;\n"
      + "  required double mydouble;\n"
      + "  required binary mybytes;\n"
      + "  required binary mystring (UTF8);\n"
      + "  required group mynestedrecord {\n"
      + "    required int32 mynestedint;\n"
      + "  }\n"
      + "  required binary myenum (ENUM);\n"
      + "  required group myarray (LIST) {\n"
      + "    repeated int32 array;\n"
      + "  }\n"
      + "  optional group myoptionalarray (LIST) {\n"
      + "    repeated int32 array;\n"
      + "  }\n"
      + "  required group myarrayofoptional (LIST) {\n"
      + "    repeated group list {\n"
      + "      optional int32 element;\n"
      + "    }\n"
      + "  }\n"
      + "  required group myrecordarray (LIST) {\n"
      + "    repeated group array {\n"
      + "      required int32 a;\n"
      + "      required int32 b;\n"
      + "    }\n"
      + "  }\n"
      + "  required group mymap (MAP) {\n"
      + "    repeated group map {\n"
      + "      required binary key (UTF8);\n"
      + "      required int32 value;\n"
      + "    }\n"
      + "  }\n"
      + "  required fixed_len_byte_array(1) myfixed;\n"
      + "}\n";

  private static final String INT96_DEPRECATED_MESSAGE =
      "INT96 is deprecated. As interim enable READ_INT96_AS_FIXED flag to read as byte array.";

  private void testAvroToParquetConversion(Schema avroSchema, String schemaString) throws Exception {
    testAvroToParquetConversion(new Configuration(false), avroSchema, schemaString);
  }

  private void testAvroToParquetConversion(Configuration conf, Schema avroSchema, String schemaString)
      throws Exception {
    AvroSchemaConverter avroSchemaConverter = new AvroSchemaConverter(conf);
    MessageType schema = avroSchemaConverter.convert(avroSchema);
    MessageType expectedMT = MessageTypeParser.parseMessageType(schemaString);
    assertThat(schema)
        .as("converting " + schema + " to " + schemaString)
        .asString()
        .isEqualTo(expectedMT.toString());
  }

  private void testParquetToAvroConversion(Schema avroSchema, String schemaString) throws Exception {
    testParquetToAvroConversion(new Configuration(false), avroSchema, schemaString);
  }

  private void testParquetToAvroConversion(Configuration conf, Schema avroSchema, String schemaString)
      throws Exception {
    AvroSchemaConverter avroSchemaConverter = new AvroSchemaConverter(conf);
    Schema schema = avroSchemaConverter.convert(MessageTypeParser.parseMessageType(schemaString));
    assertThat(schema)
        .as("converting " + schemaString + " to " + avroSchema)
        .asString()
        .isEqualTo(avroSchema.toString());
  }

  private void testRoundTripConversion(Schema avroSchema, String schemaString) throws Exception {
    testRoundTripConversion(new Configuration(), avroSchema, schemaString);
  }

  private void testRoundTripConversion(Configuration conf, Schema avroSchema, String schemaString) throws Exception {
    AvroSchemaConverter avroSchemaConverter = new AvroSchemaConverter(conf);
    MessageType schema = avroSchemaConverter.convert(avroSchema);
    MessageType expectedMT = MessageTypeParser.parseMessageType(schemaString);
    assertThat(schema)
        .as("converting " + schema + " to " + schemaString)
        .asString()
        .isEqualTo(expectedMT.toString());
    Schema convertedAvroSchema = avroSchemaConverter.convert(expectedMT);
    assertThat(convertedAvroSchema)
        .as("converting " + expectedMT + " to " + avroSchema.toString(true))
        .asString()
        .isEqualTo(avroSchema.toString());
  }

  @Test
  public void testTopLevelMustBeARecord() {
    assertThatThrownBy(() -> new AvroSchemaConverter().convert(Schema.create(INT)))
        .isInstanceOf(IllegalArgumentException.class)
        .hasMessage("Avro schema must be a record.");
  }

  @Test
  public void testAllTypes() throws Exception {
    Schema schema =
        new Schema.Parser().parse(Resources.getResource("all.avsc").openStream());
    testAvroToParquetConversion(
        NEW_BEHAVIOR,
        schema,
        "message org.apache.parquet.avro.myrecord {\n" +
            // Avro nulls are not encoded, unless they are null unions
            "  required boolean myboolean;\n"
            + "  required int32 myint;\n"
            + "  required int64 mylong;\n"
            + "  required float myfloat;\n"
            + "  required double mydouble;\n"
            + "  required binary mybytes;\n"
            + "  required binary mystring (UTF8);\n"
            + "  required group mynestedrecord {\n"
            + "    required int32 mynestedint;\n"
            + "  }\n"
            + "  required binary myenum (ENUM);\n"
            + "  required group myarray (LIST) {\n"
            + "    repeated group list {\n"
            + "      required int32 element;\n"
            + "    }\n"
            + "  }\n"
            + "  required group myemptyarray (LIST) {\n"
            + "    repeated group list {\n"
            + "      required int32 element;\n"
            + "    }\n"
            + "  }\n"
            + "  optional group myoptionalarray (LIST) {\n"
            + "    repeated group list {\n"
            + "      required int32 element;\n"
            + "    }\n"
            + "  }\n"
            + "  required group myarrayofoptional (LIST) {\n"
            + "    repeated group list {\n"
            + "      optional int32 element;\n"
            + "    }\n"
            + "  }\n"
            + "  required group mymap (MAP) {\n"
            + "    repeated group key_value {\n"
            + "      required binary key (UTF8);\n"
            + "      required int32 value;\n"
            + "    }\n"
            + "  }\n"
            + "  required group myemptymap (MAP) {\n"
            + "    repeated group key_value {\n"
            + "      required binary key (UTF8);\n"
            + "      required int32 value;\n"
            + "    }\n"
            + "  }\n"
            + "  required fixed_len_byte_array(1) myfixed;\n"
            + "}\n");
  }

  @Test
  public void testAllTypesOldListBehavior() throws Exception {
    Schema schema =
        new Schema.Parser().parse(Resources.getResource("all.avsc").openStream());
    testAvroToParquetConversion(
        schema,
        "message org.apache.parquet.avro.myrecord {\n" +
            // Avro nulls are not encoded, unless they are null unions
            "  required boolean myboolean;\n"
            + "  required int32 myint;\n"
            + "  required int64 mylong;\n"
            + "  required float myfloat;\n"
            + "  required double mydouble;\n"
            + "  required binary mybytes;\n"
            + "  required binary mystring (UTF8);\n"
            + "  required group mynestedrecord {\n"
            + "    required int32 mynestedint;\n"
            + "  }\n"
            + "  required binary myenum (ENUM);\n"
            + "  required group myarray (LIST) {\n"
            + "    repeated int32 array;\n"
            + "  }\n"
            + "  required group myemptyarray (LIST) {\n"
            + "    repeated int32 array;\n"
            + "  }\n"
            + "  optional group myoptionalarray (LIST) {\n"
            + "    repeated int32 array;\n"
            + "  }\n"
            + "  required group myarrayofoptional (LIST) {\n"
            + "    repeated int32 array;\n"
            + "  }\n"
            + "  required group mymap (MAP) {\n"
            + "    repeated group key_value {\n"
            + "      required binary key (UTF8);\n"
            + "      required int32 value;\n"
            + "    }\n"
            + "  }\n"
            + "  required group myemptymap (MAP) {\n"
            + "    repeated group key_value {\n"
            + "      required binary key (UTF8);\n"
            + "      required int32 value;\n"
            + "    }\n"
            + "  }\n"
            + "  required fixed_len_byte_array(1) myfixed;\n"
            + "}\n");
  }

  @Test
  public void testAllTypesParquetToAvro() throws Exception {
    Schema schema = new Schema.Parser()
        .parse(Resources.getResource("allFromParquetNewBehavior.avsc").openStream());
    // Cannot use round-trip assertion because enum is lost
    testParquetToAvroConversion(NEW_BEHAVIOR, schema, ALL_PARQUET_SCHEMA);
  }

  @Test
  public void testAllTypesParquetToAvroOldBehavior() throws Exception {
    Schema schema = new Schema.Parser()
        .parse(Resources.getResource("allFromParquetOldBehavior.avsc").openStream());
    // Cannot use round-trip assertion because enum is lost
    testParquetToAvroConversion(schema, ALL_PARQUET_SCHEMA);
  }

  @Test
  public void testParquetMapWithNonStringKeyFails() throws Exception {
    MessageType parquetSchema =
        MessageTypeParser.parseMessageType("message myrecord {\n" + "  required group mymap (MAP) {\n"
            + "    repeated group map (MAP_KEY_VALUE) {\n"
            + "      required int32 key;\n"
            + "      required int32 value;\n"
            + "    }\n"
            + "  }\n"
            + "}\n");
    assertThatThrownBy(() -> new AvroSchemaConverter().convert(parquetSchema))
        .isInstanceOf(IllegalArgumentException.class)
        .hasMessage("Map key type must be binary (UTF8): required int32 key");
  }

  @Test
  public void testOptionalFields() throws Exception {
    Schema schema = Schema.createRecord("record1", null, null, false);
    Schema optionalInt = optional(Schema.create(INT));
    schema.setFields(
        Collections.singletonList(new Schema.Field("myint", optionalInt, null, JsonProperties.NULL_VALUE)));
    testRoundTripConversion(schema, "message record1 {\n" + "  optional int32 myint;\n" + "}\n");
  }

  @Test
  public void testOptionalMapValue() throws Exception {
    Schema schema = Schema.createRecord("record1", null, null, false);
    Schema optionalIntMap = Schema.createMap(optional(Schema.create(INT)));
    schema.setFields(Arrays.asList(new Schema.Field("myintmap", optionalIntMap, null, null)));
    testRoundTripConversion(
        schema,
        "message record1 {\n" + "  required group myintmap (MAP) {\n"
            + "    repeated group key_value {\n"
            + "      required binary key (UTF8);\n"
            + "      optional int32 value;\n"
            + "    }\n"
            + "  }\n"
            + "}\n");
  }

  @Test
  public void testOptionalArrayElement() throws Exception {
    Schema schema = Schema.createRecord("record1", null, null, false);
    Schema optionalIntArray = Schema.createArray(optional(Schema.create(INT)));
    schema.setFields(Arrays.asList(new Schema.Field("myintarray", optionalIntArray, null, null)));
    testRoundTripConversion(
        NEW_BEHAVIOR,
        schema,
        "message record1 {\n" + "  required group myintarray (LIST) {\n"
            + "    repeated group list {\n"
            + "      optional int32 element;\n"
            + "    }\n"
            + "  }\n"
            + "}\n");
  }

  @Test
  public void testUnionOfTwoTypes() throws Exception {
    Schema schema = Schema.createRecord("record2", null, null, false);
    Schema multipleTypes = Schema.createUnion(
        Arrays.asList(Schema.create(Schema.Type.NULL), Schema.create(INT), Schema.create(Schema.Type.FLOAT)));
    schema.setFields(Arrays.asList(new Schema.Field("myunion", multipleTypes, null, JsonProperties.NULL_VALUE)));

    // Avro union is modelled using optional data members of the different
    // types. This does not translate back into an Avro union
    testAvroToParquetConversion(
        schema,
        "message record2 {\n" + "  optional group myunion {\n"
            + "    optional int32 member0;\n"
            + "    optional float member1;\n"
            + "  }\n"
            + "}\n");
  }

  @Test
  public void testArrayOfOptionalRecords() throws Exception {
    Schema innerRecord = Schema.createRecord("element", null, null, false);
    Schema optionalString = optional(Schema.create(Schema.Type.STRING));
    innerRecord.setFields(Lists.newArrayList(
        new Schema.Field("s1", optionalString, null, JsonProperties.NULL_VALUE),
        new Schema.Field("s2", optionalString, null, JsonProperties.NULL_VALUE)));
    Schema schema = Schema.createRecord("HasArray", null, null, false);
    schema.setFields(
        Lists.newArrayList(new Schema.Field("myarray", Schema.createArray(optional(innerRecord)), null, null)));

    testRoundTripConversion(
        NEW_BEHAVIOR,
        schema,
        "message HasArray {\n" + "  required group myarray (LIST) {\n"
            + "    repeated group list {\n"
            + "      optional group element {\n"
            + "        optional binary s1 (UTF8);\n"
            + "        optional binary s2 (UTF8);\n"
            + "      }\n"
            + "    }\n"
            + "  }\n"
            + "}\n");
  }

  @Test
  public void testArrayOfOptionalRecordsOldBehavior() throws Exception {
    Schema innerRecord = Schema.createRecord("InnerRecord", null, null, false);
    Schema optionalString = optional(Schema.create(Schema.Type.STRING));
    innerRecord.setFields(Lists.newArrayList(
        new Schema.Field("s1", optionalString, null, JsonProperties.NULL_VALUE),
        new Schema.Field("s2", optionalString, null, JsonProperties.NULL_VALUE)));
    Schema schema = Schema.createRecord("HasArray", null, null, false);
    schema.setFields(
        Lists.newArrayList(new Schema.Field("myarray", Schema.createArray(optional(innerRecord)), null, null)));

    // Cannot use round-trip assertion because InnerRecord optional is removed
    testAvroToParquetConversion(
        schema,
        "message HasArray {\n" + "  required group myarray (LIST) {\n"
            + "    repeated group array {\n"
            + "      optional binary s1 (UTF8);\n"
            + "      optional binary s2 (UTF8);\n"
            + "    }\n"
            + "  }\n"
            + "}\n");
  }

  @Test
  public void testOldAvroListOfLists() throws Exception {
    Schema listOfLists = optional(Schema.createArray(Schema.createArray(Schema.create(INT))));
    Schema schema = Schema.createRecord("AvroCompatListInList", null, null, false);
    schema.setFields(
        Lists.newArrayList(new Schema.Field("listOfLists", listOfLists, null, JsonProperties.NULL_VALUE)));

    testRoundTripConversion(
        schema,
        "message AvroCompatListInList {\n" + "  optional group listOfLists (LIST) {\n"
            + "    repeated group array (LIST) {\n"
            + "      repeated int32 array;\n"
            + "    }\n"
            + "  }\n"
            + "}");
    // Cannot use round-trip assertion because 3-level representation is used
    testParquetToAvroConversion(
        NEW_BEHAVIOR,
        schema,
        "message AvroCompatListInList {\n" + "  optional group listOfLists (LIST) {\n"
            + "    repeated group array (LIST) {\n"
            + "      repeated int32 array;\n"
            + "    }\n"
            + "  }\n"
            + "}");
  }

  @Test
  public void testConvertUngroupedRepeatedField() throws Exception {
    testParquetToAvroConversion(
        NEW_BEHAVIOR,
        new Schema.Parser()
            .parse("{\"type\": \"record\","
                + "  \"name\": \"SchemaWithRepeatedField\","
                + "  \"fields\": [{"
                + "    \"name\": \"repeatedField\","
                + "    \"type\": {\"type\": \"array\",\"items\": \"int\"},"
                + "    \"default\": []"
                + "  }]"
                + "}"),
        "message SchemaWithRepeatedField { repeated int32 repeatedField; }");
  }

  @Test
  public void testOldThriftListOfLists() throws Exception {
    Schema listOfLists = optional(Schema.createArray(Schema.createArray(Schema.create(INT))));
    Schema schema = Schema.createRecord("ThriftCompatListInList", null, null, false);
    schema.setFields(
        Lists.newArrayList(new Schema.Field("listOfLists", listOfLists, null, JsonProperties.NULL_VALUE)));

    // Cannot use round-trip assertion because repeated group names differ
    testParquetToAvroConversion(
        schema,
        "message ThriftCompatListInList {\n" + "  optional group listOfLists (LIST) {\n"
            + "    repeated group listOfLists_tuple (LIST) {\n"
            + "      repeated int32 listOfLists_tuple_tuple;\n"
            + "    }\n"
            + "  }\n"
            + "}");
    // Cannot use round-trip assertion because 3-level representation is used
    testParquetToAvroConversion(
        NEW_BEHAVIOR,
        schema,
        "message ThriftCompatListInList {\n" + "  optional group listOfLists (LIST) {\n"
            + "    repeated group listOfLists_tuple (LIST) {\n"
            + "      repeated int32 listOfLists_tuple_tuple;\n"
            + "    }\n"
            + "  }\n"
            + "}");
  }

  @Test
  public void testUnknownTwoLevelListOfLists() throws Exception {
    // This tests the case where we don't detect a 2-level list by the repeated
    // group's name, but it must be 2-level because the repeated group doesn't
    // contain an optional or repeated element as required for 3-level lists
    Schema listOfLists = optional(Schema.createArray(Schema.createArray(Schema.create(INT))));
    Schema schema = Schema.createRecord("UnknownTwoLevelListInList", null, null, false);
    schema.setFields(
        Lists.newArrayList(new Schema.Field("listOfLists", listOfLists, null, JsonProperties.NULL_VALUE)));

    // Cannot use round-trip assertion because repeated group names differ
    testParquetToAvroConversion(
        schema,
        "message UnknownTwoLevelListInList {\n" + "  optional group listOfLists (LIST) {\n"
            + "    repeated group mylist (LIST) {\n"
            + "      repeated int32 innerlist;\n"
            + "    }\n"
            + "  }\n"
            + "}");
    // Cannot use round-trip assertion because 3-level representation is used
    testParquetToAvroConversion(
        NEW_BEHAVIOR,
        schema,
        "message UnknownTwoLevelListInList {\n" + "  optional group listOfLists (LIST) {\n"
            + "    repeated group mylist (LIST) {\n"
            + "      repeated int32 innerlist;\n"
            + "    }\n"
            + "  }\n"
            + "}");
  }

  @Test
  public void testParquetMapWithoutMapKeyValueAnnotation() throws Exception {
    Schema schema = Schema.createRecord("myrecord", null, null, false);
    Schema map = Schema.createMap(Schema.create(INT));
    schema.setFields(Collections.singletonList(new Schema.Field("mymap", map, null, null)));
    String parquetSchema = "message myrecord {\n" + "  required group mymap (MAP) {\n"
        + "    repeated group map {\n"
        + "      required binary key (UTF8);\n"
        + "      required int32 value;\n"
        + "    }\n"
        + "  }\n"
        + "}\n";

    testParquetToAvroConversion(schema, parquetSchema);
    testParquetToAvroConversion(NEW_BEHAVIOR, schema, parquetSchema);
  }

  @Test
  public void testDecimalBytesType() throws Exception {
    Schema schema = Schema.createRecord("myrecord", null, null, false);
    Schema decimal = LogicalTypes.decimal(9, 2).addToSchema(Schema.create(Schema.Type.BYTES));
    schema.setFields(Collections.singletonList(new Schema.Field("dec", decimal, null, null)));

    testRoundTripConversion(schema, "message myrecord {\n" + "  required binary dec (DECIMAL(9,2));\n" + "}\n");
  }

  @Test
  public void testDecimalFixedType() throws Exception {
    Schema schema = Schema.createRecord("myrecord", null, null, false);
    Schema decimal = LogicalTypes.decimal(9, 2).addToSchema(Schema.createFixed("dec", null, null, 8));
    schema.setFields(Collections.singletonList(new Schema.Field("dec", decimal, null, null)));

    testRoundTripConversion(
        schema, "message myrecord {\n" + "  required fixed_len_byte_array(8) dec (DECIMAL(9,2));\n" + "}\n");
  }

  @Test
  public void testDecimalIntegerType() throws Exception {
    Schema expected = Schema.createRecord(
        "myrecord", null, null, false, Arrays.asList(new Schema.Field("dec", Schema.create(INT), null, null)));

    // the decimal portion is lost because it isn't valid in Avro
    testParquetToAvroConversion(
        expected, "message myrecord {\n" + "  required int32 dec (DECIMAL(9,2));\n" + "}\n");
  }

  @Test
  public void testDecimalLongType() throws Exception {
    Schema expected = Schema.createRecord(
        "myrecord", null, null, false, Arrays.asList(new Schema.Field("dec", Schema.create(LONG), null, null)));

    // the decimal portion is lost because it isn't valid in Avro
    testParquetToAvroConversion(
        expected, "message myrecord {\n" + "  required int64 dec (DECIMAL(9,2));\n" + "}\n");
  }

  @Test
  public void testParquetInt96AsFixed12AvroType() throws Exception {
    Configuration enableInt96ReadingConfig = new Configuration();
    enableInt96ReadingConfig.setBoolean(AvroReadSupport.READ_INT96_AS_FIXED, true);

    Schema schema = Schema.createRecord("myrecord", null, null, false);
    Schema int96schema = Schema.createFixed("int96_field", "INT96 represented as byte[12]", null, 12);
    schema.setFields(Collections.singletonList(new Schema.Field("int96_field", int96schema, null, null)));

    testParquetToAvroConversion(
        enableInt96ReadingConfig, schema, "message myrecord {\n" + "  required int96 int96_field;\n" + "}\n");
  }

  @Test
  public void testParquetInt96DefaultFail() throws Exception {
    Schema schema = Schema.createRecord("myrecord", null, null, false);

    MessageType parquetSchemaWithInt96 =
        MessageTypeParser.parseMessageType("message myrecord {\n  required int96 int96_field;\n}\n");

    assertThatThrownBy(() -> new AvroSchemaConverter().convert(parquetSchemaWithInt96))
        .isInstanceOf(IllegalArgumentException.class)
        .hasMessage(INT96_DEPRECATED_MESSAGE);
  }

  @Test
  public void testMultipleInt96FieldsToStringConversion() throws Exception {
    Configuration enableInt96ReadingConfig = new Configuration();
    enableInt96ReadingConfig.setBoolean(AvroReadSupport.READ_INT96_AS_FIXED, true);

    Types.MessageTypeBuilder builder = Types.buildMessage();
    builder.optional(PrimitiveType.PrimitiveTypeName.INT96).named("timestamp_1");
    builder.optional(PrimitiveType.PrimitiveTypeName.INT96).named("timestamp_2");
    MessageType int96Schema = builder.named("int96Schema");

    AvroSchemaConverter converter = new AvroSchemaConverter(enableInt96ReadingConfig);
    Schema avroSchema = converter.convert(int96Schema);

    String schemaString = avroSchema.toString(true);

    assertThat(schemaString)
        .as("First field should have full timestamp_1 definition")
        .contains("\"name\" : \"timestamp_1\"");
    assertThat(schemaString)
        .as("Second field should have full timestamp_2 definition")
        .contains("\"name\" : \"timestamp_2\"");

    assertThat(schemaString)
        .as("Should not reference bare 'INT96' type anymore")
        .doesNotContain("\"type\" : [ \"null\", \"INT96\" ]");
  }

  @Test
  public void testDateType() throws Exception {
    Schema date = LogicalTypes.date().addToSchema(Schema.create(INT));
    Schema expected = Schema.createRecord(
        "myrecord", null, null, false, Arrays.asList(new Schema.Field("date", date, null, null)));

    testRoundTripConversion(expected, "message myrecord {\n" + "  required int32 date (DATE);\n" + "}\n");

    for (PrimitiveTypeName primitive :
        new PrimitiveTypeName[] {INT64, INT96, FLOAT, DOUBLE, BOOLEAN, BINARY, FIXED_LEN_BYTE_ARRAY}) {
      final PrimitiveType type;
      if (primitive == FIXED_LEN_BYTE_ARRAY) {
        type = new PrimitiveType(REQUIRED, primitive, 12, "test", DATE);
      } else {
        type = new PrimitiveType(REQUIRED, primitive, "test", DATE);
      }

      final String expectedMessage =
          primitive == INT96 ? INT96_DEPRECATED_MESSAGE : "Date can only be used with an underlying int type";
      assertThatThrownBy(() -> new AvroSchemaConverter().convert(message(type)))
          .isInstanceOf(IllegalArgumentException.class)
          .hasMessage(expectedMessage);
    }
  }

  @Test
  public void testTimeMillisType() throws Exception {
    Schema date = LogicalTypes.timeMillis().addToSchema(Schema.create(INT));
    Schema expected = Schema.createRecord(
        "myrecord", null, null, false, Arrays.asList(new Schema.Field("time", date, null, null)));

    testRoundTripConversion(
        expected, "message myrecord {\n" + "  required int32 time (TIME(MILLIS,true));\n" + "}\n");

    for (PrimitiveTypeName primitive :
        new PrimitiveTypeName[] {INT64, INT96, FLOAT, DOUBLE, BOOLEAN, BINARY, FIXED_LEN_BYTE_ARRAY}) {
      final PrimitiveType type;
      if (primitive == FIXED_LEN_BYTE_ARRAY) {
        type = new PrimitiveType(REQUIRED, primitive, 12, "test", TIME_MILLIS);
      } else {
        type = new PrimitiveType(REQUIRED, primitive, "test", TIME_MILLIS);
      }

      final String expectedMessage = primitive == INT96
          ? INT96_DEPRECATED_MESSAGE
          : "Time (millis) can only be used with an underlying int type";
      assertThatThrownBy(() -> new AvroSchemaConverter().convert(message(type)))
          .isInstanceOf(IllegalArgumentException.class)
          .hasMessage(expectedMessage);
    }
  }

  @Test
  public void testTimeMicrosType() throws Exception {
    Schema date = LogicalTypes.timeMicros().addToSchema(Schema.create(LONG));
    Schema expected = Schema.createRecord(
        "myrecord", null, null, false, Arrays.asList(new Schema.Field("time", date, null, null)));

    testRoundTripConversion(
        expected, "message myrecord {\n" + "  required int64 time (TIME(MICROS,true));\n" + "}\n");

    for (PrimitiveTypeName primitive :
        new PrimitiveTypeName[] {INT32, INT96, FLOAT, DOUBLE, BOOLEAN, BINARY, FIXED_LEN_BYTE_ARRAY}) {
      final PrimitiveType type;
      if (primitive == FIXED_LEN_BYTE_ARRAY) {
        type = new PrimitiveType(REQUIRED, primitive, 12, "test", TIME_MICROS);
      } else {
        type = new PrimitiveType(REQUIRED, primitive, "test", TIME_MICROS);
      }

      final String expectedMessage = primitive == INT96
          ? INT96_DEPRECATED_MESSAGE
          : "Time (micros) can only be used with an underlying long type";
      assertThatThrownBy(() -> new AvroSchemaConverter().convert(message(type)))
          .isInstanceOf(IllegalArgumentException.class)
          .hasMessage(expectedMessage);
    }
  }

  @Test
  public void testTimestampMillisType() throws Exception {
    Schema date = LogicalTypes.timestampMillis().addToSchema(Schema.create(LONG));
    Schema expected = Schema.createRecord(
        "myrecord", null, null, false, Arrays.asList(new Schema.Field("timestamp", date, null, null)));

    testRoundTripConversion(
        expected, "message myrecord {\n" + "  required int64 timestamp (TIMESTAMP(MILLIS,true));\n" + "}\n");

    // Test that conversions for timestamp types only use APIs that are available in the user's Avro version
    for (String avroVersion : ImmutableSet.of("1.7.0", "1.8.0", "1.9.0", "1.10.0", "1.11.0")) {
      avroRecordConverterMock
          .when(AvroRecordConverter::getRuntimeAvroVersion)
          .thenReturn(avroVersion);
      final Schema converted = new AvroSchemaConverter()
          .convert(Types.buildMessage()
              .addField(Types.primitive(INT64, Type.Repetition.REQUIRED)
                  .as(LogicalTypeAnnotation.timestampType(
                      false, LogicalTypeAnnotation.TimeUnit.MILLIS))
                  .length(1)
                  .named("timestamp_type"))
              .named("TestAvro"));

      assertThat(converted
              .getField("timestamp_type")
              .schema()
              .getLogicalType()
              .getName())
          .isEqualTo(avroVersion.matches("1\\.[789]\\.\\d+") ? "timestamp-millis" : "local-timestamp-millis");
    }

    for (PrimitiveTypeName primitive :
        new PrimitiveTypeName[] {INT32, INT96, FLOAT, DOUBLE, BOOLEAN, BINARY, FIXED_LEN_BYTE_ARRAY}) {
      final PrimitiveType type;
      if (primitive == FIXED_LEN_BYTE_ARRAY) {
        type = new PrimitiveType(REQUIRED, primitive, 12, "test", TIMESTAMP_MILLIS);
      } else {
        type = new PrimitiveType(REQUIRED, primitive, "test", TIMESTAMP_MILLIS);
      }

      final String expectedMessage = primitive == INT96
          ? INT96_DEPRECATED_MESSAGE
          : "Timestamp (millis) can only be used with an underlying long type";
      assertThatThrownBy(() -> new AvroSchemaConverter().convert(message(type)))
          .isInstanceOf(IllegalArgumentException.class)
          .hasMessage(expectedMessage);
    }
  }

  @Test
  public void testLocalTimestampMillisType() throws Exception {
    Schema date = LogicalTypes.localTimestampMillis().addToSchema(Schema.create(LONG));
    Schema expected = Schema.createRecord(
        "myrecord", null, null, false, Arrays.asList(new Schema.Field("timestamp", date, null, null)));

    testRoundTripConversion(
        expected, "message myrecord {\n" + "  required int64 timestamp (TIMESTAMP(MILLIS,false));\n" + "}\n");

    for (PrimitiveTypeName primitive :
        new PrimitiveTypeName[] {INT32, INT96, FLOAT, DOUBLE, BOOLEAN, BINARY, FIXED_LEN_BYTE_ARRAY}) {
      final PrimitiveType type;
      if (primitive == FIXED_LEN_BYTE_ARRAY) {
        type = new PrimitiveType(REQUIRED, primitive, 12, "test", TIMESTAMP_MILLIS);
      } else {
        type = new PrimitiveType(REQUIRED, primitive, "test", TIMESTAMP_MILLIS);
      }

      final String expectedMessage = primitive == INT96
          ? INT96_DEPRECATED_MESSAGE
          : "Timestamp (millis) can only be used with an underlying long type";
      assertThatThrownBy(() -> new AvroSchemaConverter().convert(message(type)))
          .isInstanceOf(IllegalArgumentException.class)
          .hasMessage(expectedMessage);
    }
  }

  @Test
  public void testTimestampMicrosType() throws Exception {
    Schema date = LogicalTypes.timestampMicros().addToSchema(Schema.create(LONG));
    Schema expected = Schema.createRecord(
        "myrecord", null, null, false, Arrays.asList(new Schema.Field("timestamp", date, null, null)));

    testRoundTripConversion(
        expected, "message myrecord {\n" + "  required int64 timestamp (TIMESTAMP(MICROS,true));\n" + "}\n");

    for (PrimitiveTypeName primitive :
        new PrimitiveTypeName[] {INT32, INT96, FLOAT, DOUBLE, BOOLEAN, BINARY, FIXED_LEN_BYTE_ARRAY}) {
      final PrimitiveType type;
      if (primitive == FIXED_LEN_BYTE_ARRAY) {
        type = new PrimitiveType(REQUIRED, primitive, 12, "test", TIMESTAMP_MICROS);
      } else {
        type = new PrimitiveType(REQUIRED, primitive, "test", TIMESTAMP_MICROS);
      }

      final String expectedMessage = primitive == INT96
          ? INT96_DEPRECATED_MESSAGE
          : "Timestamp (micros) can only be used with an underlying long type";
      assertThatThrownBy(() -> new AvroSchemaConverter().convert(message(type)))
          .isInstanceOf(IllegalArgumentException.class)
          .hasMessage(expectedMessage);
    }

    // Test that conversions for timestamp types only use APIs that are available in the user's Avro version
    for (String avroVersion : ImmutableSet.of("1.7.0", "1.8.0", "1.9.0", "1.10.0", "1.11.0")) {
      avroRecordConverterMock
          .when(AvroRecordConverter::getRuntimeAvroVersion)
          .thenReturn(avroVersion);
      final Schema converted = new AvroSchemaConverter()
          .convert(Types.buildMessage()
              .addField(Types.primitive(INT64, Type.Repetition.REQUIRED)
                  .as(LogicalTypeAnnotation.timestampType(
                      false, LogicalTypeAnnotation.TimeUnit.MICROS))
                  .length(1)
                  .named("timestamp_type"))
              .named("TestAvro"));

      assertThat(converted
              .getField("timestamp_type")
              .schema()
              .getLogicalType()
              .getName())
          .isEqualTo(avroVersion.matches("1\\.[789]\\.\\d+") ? "timestamp-micros" : "local-timestamp-micros");
    }
  }

  @Test
  public void testLocalTimestampMicrosType() throws Exception {
    Schema date = LogicalTypes.localTimestampMicros().addToSchema(Schema.create(LONG));
    Schema expected = Schema.createRecord(
        "myrecord", null, null, false, Arrays.asList(new Schema.Field("timestamp", date, null, null)));

    testRoundTripConversion(
        expected, "message myrecord {\n" + "  required int64 timestamp (TIMESTAMP(MICROS,false));\n" + "}\n");

    for (PrimitiveTypeName primitive :
        new PrimitiveTypeName[] {INT32, INT96, FLOAT, DOUBLE, BOOLEAN, BINARY, FIXED_LEN_BYTE_ARRAY}) {
      final PrimitiveType type;
      if (primitive == FIXED_LEN_BYTE_ARRAY) {
        type = new PrimitiveType(REQUIRED, primitive, 12, "test", TIMESTAMP_MICROS);
      } else {
        type = new PrimitiveType(REQUIRED, primitive, "test", TIMESTAMP_MICROS);
      }

      final String expectedMessage = primitive == INT96
          ? INT96_DEPRECATED_MESSAGE
          : "Timestamp (micros) can only be used with an underlying long type";
      assertThatThrownBy(() -> new AvroSchemaConverter().convert(message(type)))
          .isInstanceOf(IllegalArgumentException.class)
          .hasMessage(expectedMessage);
    }
  }

  @Test
  public void testReuseNameInNestedStructure() throws Exception {
    Schema innerA1 = record("a1", "a12", field("a4", primitive(Schema.Type.FLOAT)));

    Schema outerA1 = record("a1", field("a2", primitive(Schema.Type.FLOAT)), optionalField("a1", innerA1));
    Schema schema = record("Message", optionalField("a1", outerA1));

    String parquetSchema = "message Message {\n"
        + "  optional group a1 {\n"
        + "    required float a2;\n"
        + "    optional group a1 {\n"
        + "      required float a4;\n"
        + "     }\n"
        + "  }\n"
        + "}\n";

    testParquetToAvroConversion(schema, parquetSchema);
    testParquetToAvroConversion(NEW_BEHAVIOR, schema, parquetSchema);
  }

  @Test
  public void testReuseNameInNestedStructureAtSameLevel() throws Exception {
    Schema a2 = record("a2", field("a4", primitive(Schema.Type.FLOAT)));
    Schema a22 = record(
        "a2", "a22", field("a4", primitive(Schema.Type.FLOAT)), field("a5", primitive(Schema.Type.FLOAT)));

    Schema a1 = record("a1", optionalField("a2", a2));
    Schema a3 = record("a3", optionalField("a2", a22));

    Schema schema = record("Message", optionalField("a1", a1), optionalField("a3", a3));

    String parquetSchema = "message Message {\n"
        + "  optional group a1 {\n"
        + "    optional group a2 {\n"
        + "      required float a4;\n"
        + "     }\n"
        + "  }\n"
        + "  optional group a3 {\n"
        + "    optional group a2 {\n"
        + "      required float a4;\n"
        + "      required float a5;\n"
        + "     }\n"
        + "  }\n"
        + "}\n";

    testParquetToAvroConversion(schema, parquetSchema);
    testParquetToAvroConversion(NEW_BEHAVIOR, schema, parquetSchema);
  }

  @Test
  public void testReuseNamesArrays() throws Exception {
    Schema a1 = record("array", field("a4", primitive(Schema.Type.FLOAT)));
    Schema a2 = Schema.createFixed("array", null, "array2", 1);
    Schema a3 = Schema.createFixed("array", null, "array3", 1);
    Schema schema = record("Message", field("a1", array(a1)), field("a2", array(a2)), field("a3", array(a3)));

    String parquetSchema = "message Message {\n"
        + "  required group a1 (LIST) {\n"
        + "    repeated group array {\n"
        + "      required float a4;\n"
        + "    }\n"
        + "  }\n"
        + "  required group a2 (LIST) {\n"
        + "    repeated fixed_len_byte_array(1) array;\n"
        + "  }\n"
        + "  required group a3 (LIST) {\n"
        + "    repeated fixed_len_byte_array(1) array;\n"
        + "  }\n"
        + "}\n";

    testParquetToAvroConversion(schema, parquetSchema);
    testParquetToAvroConversion(NEW_BEHAVIOR, schema, parquetSchema);
  }

  @Test
  public void testUUIDType() throws Exception {
    Schema fromAvro = Schema.createRecord(
        "myrecord",
        null,
        null,
        false,
        Arrays.asList(
            new Schema.Field("uuid", LogicalTypes.uuid().addToSchema(Schema.create(STRING)), null, null)));
    String parquet = "message myrecord {\n" + "  required binary uuid (STRING);\n" + "}\n";
    Schema toAvro = Schema.createRecord(
        "myrecord",
        null,
        null,
        false,
        Arrays.asList(new Schema.Field("uuid", Schema.create(STRING), null, null)));

    testAvroToParquetConversion(fromAvro, parquet);
    testParquetToAvroConversion(toAvro, parquet);

    assertThat(checkReaderWriterCompatibility(fromAvro, toAvro).getType()).isEqualTo(COMPATIBLE);
  }

  @Test
  public void testUUIDTypeWithParquetUUID() throws Exception {
    Schema uuid = LogicalTypes.uuid().addToSchema(Schema.create(STRING));
    Schema expected = Schema.createRecord(
        "myrecord", null, null, false, Arrays.asList(new Schema.Field("uuid", uuid, null, null)));

    testRoundTripConversion(
        AvroTestUtil.conf(AvroWriteSupport.WRITE_PARQUET_UUID, true),
        expected,
        "message myrecord {\n" + "  required fixed_len_byte_array(16) uuid (UUID);\n" + "}\n");
  }

  @Test
  public void testAvroFixed12AsParquetInt96Type() throws Exception {
    Schema schema = new Schema.Parser()
        .parse(Resources.getResource("fixedToInt96.avsc").openStream());

    Configuration conf = new Configuration();
    conf.setStrings(
        WRITE_FIXED_AS_INT96,
        "int96",
        "mynestedrecord.int96inrecord",
        "mynestedrecord.myarrayofoptional",
        "mynestedrecord.mymap");
    testAvroToParquetConversion(
        conf,
        schema,
        "message org.apache.parquet.avro.fixedToInt96 {\n"
            + "  required int96 int96;\n"
            + "  required fixed_len_byte_array(12) notanint96;\n"
            + "  required group mynestedrecord {\n"
            + "    required int96 int96inrecord;\n"
            + "    required group myarrayofoptional (LIST) {\n"
            + "      repeated int96 array;\n"
            + "    }\n"
            + "    required group mymap (MAP) {\n"
            + "      repeated group key_value {\n"
            + "        required binary key (STRING);\n"
            + "        required int96 value;\n"
            + "      }\n"
            + "    }\n"
            + "  }\n"
            + "  required fixed_len_byte_array(1) onebytefixed;\n"
            + "}");

    conf.setStrings(WRITE_FIXED_AS_INT96, "onebytefixed");
    assertThatThrownBy(() -> new AvroSchemaConverter(conf).convert(schema))
        .isInstanceOf(IllegalArgumentException.class)
        .hasMessage("The size of the fixed type field onebytefixed must be 12 bytes for INT96 conversion");
  }

  @Test
  public void testRecursiveSchemaThrowsException() {
    String recursiveSchemaJson = "{"
        + "\"type\": \"record\", \"name\": \"Node\", \"fields\": ["
        + "  {\"name\": \"value\", \"type\": \"int\"},"
        + "  {\"name\": \"children\", \"type\": ["
        + "    \"null\", {"
        + "      \"type\": \"array\", \"items\": [\"null\", \"Node\"]"
        + "    }"
        + "  ], \"default\": null}"
        + "]}";

    Schema recursiveSchema = new Schema.Parser().parse(recursiveSchemaJson);

    assertThatThrownBy(() -> new AvroSchemaConverter().convert(recursiveSchema))
        .isInstanceOf(UnsupportedOperationException.class)
        .hasMessage("Recursive Avro schemas are not supported by parquet-avro: Node");
  }

  @Test
  public void testRecursiveSchemaFromGitHubIssue() {
    String issueSchemaJson = "{"
        + "\"type\": \"record\", \"name\": \"ObjXX\", \"fields\": ["
        + "  {\"name\": \"id\", \"type\": [\"null\", \"long\"], \"default\": null},"
        + "  {\"name\": \"struct_add_list\", \"type\": [\"null\", {"
        + "    \"type\": \"array\", \"items\": [\"null\", {"
        + "      \"type\": \"record\", \"name\": \"ObjStructAdd\", \"fields\": ["
        + "        {\"name\": \"name\", \"type\": [\"null\", \"string\"], \"default\": null},"
        + "        {\"name\": \"fld_list\", \"type\": [\"null\", {"
        + "          \"type\": \"array\", \"items\": [\"null\", {"
        + "            \"type\": \"record\", \"name\": \"ObjStructAddFld\", \"fields\": ["
        + "              {\"name\": \"name\", \"type\": [\"null\", \"string\"], \"default\": null},"
        + "              {\"name\": \"ref_val\", \"type\": [\"null\", \"ObjStructAdd\"], \"default\": null}"
        + "            ]"
        + "          }]"
        + "        }], \"default\": null}"
        + "      ]"
        + "    }]"
        + "  }], \"default\": null},"
        + "  {\"name\": \"kafka_timestamp\", \"type\": {\"type\": \"long\", \"logicalType\": \"timestamp-millis\"}}"
        + "]}";

    Schema issueSchema = new Schema.Parser().parse(issueSchemaJson);

    assertThatThrownBy(() -> new AvroSchemaConverter().convert(issueSchema))
        .isInstanceOf(UnsupportedOperationException.class)
        .hasMessage("Recursive Avro schemas are not supported by parquet-avro: ObjStructAdd");
  }

  @Test
  public void testRecursiveSchemaErrorMessage() {
    String recursiveSchemaJson = "{"
        + "\"type\": \"record\", \"name\": \"TestRecord\", \"fields\": ["
        + "  {\"name\": \"self\", \"type\": [\"null\", \"TestRecord\"], \"default\": null}"
        + "]}";

    Schema recursiveSchema = new Schema.Parser().parse(recursiveSchemaJson);

    // With our cycle detection fix, this should throw UnsupportedOperationException
    assertThatThrownBy(() -> new AvroSchemaConverter().convert(recursiveSchema))
        .isInstanceOf(UnsupportedOperationException.class)
        .hasMessage("Recursive Avro schemas are not supported by parquet-avro: TestRecord");
  }

  @Test
  public void testDeeplyNestedNonRecursiveSchema() {
    Schema level3 = record("Level3", field("value", primitive(STRING)));
    Schema level2 = record("Level2", field("level3", level3));
    Schema level1 = record("Level1", field("level2", level2));
    Schema rootSchema = record("Root", field("level1", level1));

    AvroSchemaConverter converter = new AvroSchemaConverter();
    MessageType result = converter.convert(rootSchema);
    assertThat(result)
        .as("Non-recursive deep schema should convert successfully")
        .isNotNull();
    assertThat(result.getName()).as("Root schema name should be preserved").isEqualTo("Root");
  }

  public static Schema optional(Schema original) {
    return Schema.createUnion(Lists.newArrayList(Schema.create(Schema.Type.NULL), original));
  }

  public static MessageType message(PrimitiveType primitive) {
    return Types.buildMessage().addField(primitive).named("myrecord");
  }
}
