locations.geojson POC phase2: End-to-end partial support of json data

.gitignore

@@ -104,3 +104,4 @@ app/pkg/bin/
 processor/notices/bin/
 processor/notices/tests/bin/
 web/service/bin/
+/web/service/execution_result.json

core/src/main/java/org/mobilitydata/gtfsvalidator/notice/UniqueLocationIdViolationNotice.java

@@ -0,0 +1,61 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.mobilitydata.gtfsvalidator.notice;
+
+import static org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice.SectionRef.FILE_REQUIREMENTS;
+import static org.mobilitydata.gtfsvalidator.notice.SeverityLevel.ERROR;
+
+import org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice;
+import org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice.SectionRefs;
+import org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice.UrlRef;
+
+/**
+ * Feature id from locations.geojson already used.
+ *
+ * <p>The id of one of the features of the locations.geojson file already exists in stops.txt or
+ * location_groups.txt
+ */
+@GtfsValidationNotice(
+    severity = ERROR,
+    sections = @SectionRefs(FILE_REQUIREMENTS),
+    urls = {
+      @UrlRef(
+          label = "Original Python validator implementation",
+          url = "https://github.com/google/transitfeed")
+    })
+public class UniqueLocationIdViolationNotice extends ValidationNotice {
+
+  /** The id that already exists. */
+  private final String id;
+
+  /** The name of the file that already has this id. */
+  private final String fileWithIdAlreadyPresent;
+
+  /** The name of the field that contains this id. */
+  private final String fieldNameInFile;
+
+  /** The row of the record in the file where the id is already present. */
+  private final int csvRowNumber;
+
+  public UniqueLocationIdViolationNotice(
+      String id, String fileWithIdAlreadyPresent, String fieldNameInFile, int csvRowNumber) {
+
+    this.id = id;
+    this.fileWithIdAlreadyPresent = fileWithIdAlreadyPresent;
+    this.fieldNameInFile = fieldNameInFile;
+    this.csvRowNumber = csvRowNumber;
+  }
+}

core/src/main/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoader.java

@@ -61,22 +61,19 @@ public static GtfsTableContainer load(
       csvFile = new CsvFile(csvInputStream, gtfsFilename, settings);
     } catch (TextParsingException e) {
       noticeContainer.addValidationNotice(new CsvParsingFailedNotice(gtfsFilename, e));
-      return tableDescriptor.createContainerForInvalidStatus(
-          GtfsTableContainer.TableStatus.INVALID_HEADERS);
+      return tableDescriptor.createContainerForInvalidStatus(TableStatus.INVALID_HEADERS);
     }
     if (csvFile.isEmpty()) {
       noticeContainer.addValidationNotice(new EmptyFileNotice(gtfsFilename));
-      return tableDescriptor.createContainerForInvalidStatus(
-          GtfsTableContainer.TableStatus.EMPTY_FILE);
+      return tableDescriptor.createContainerForInvalidStatus(TableStatus.EMPTY_FILE);
     }
     final CsvHeader header = csvFile.getHeader();
     final ImmutableList<GtfsColumnDescriptor> columnDescriptors = tableDescriptor.getColumns();
     final NoticeContainer headerNotices =
         validateHeaders(validatorProvider, gtfsFilename, header, columnDescriptors);
     noticeContainer.addAll(headerNotices);
     if (headerNotices.hasValidationErrors()) {
-      return tableDescriptor.createContainerForInvalidStatus(
-          GtfsTableContainer.TableStatus.INVALID_HEADERS);
+      return tableDescriptor.createContainerForInvalidStatus(TableStatus.INVALID_HEADERS);
     }
     final int nColumns = columnDescriptors.size();
     final ImmutableMap<String, GtfsFieldLoader> fieldLoadersMap = tableDescriptor.getFieldLoaders();
@@ -133,15 +130,13 @@ public static GtfsTableContainer load(
       }
     } catch (TextParsingException e) {
       noticeContainer.addValidationNotice(new CsvParsingFailedNotice(gtfsFilename, e));
-      return tableDescriptor.createContainerForInvalidStatus(
-          GtfsTableContainer.TableStatus.UNPARSABLE_ROWS);
+      return tableDescriptor.createContainerForInvalidStatus(TableStatus.UNPARSABLE_ROWS);
     } finally {
       logFieldCacheStats(gtfsFilename, fieldCaches, columnDescriptors);
     }
     if (hasUnparsableRows) {
       logger.atSevere().log("Failed to parse some rows in %s", gtfsFilename);
-      return tableDescriptor.createContainerForInvalidStatus(
-          GtfsTableContainer.TableStatus.UNPARSABLE_ROWS);
+      return tableDescriptor.createContainerForInvalidStatus(TableStatus.UNPARSABLE_ROWS);
     }
     GtfsTableContainer table =
         tableDescriptor.createContainerForHeaderAndEntities(header, entities, noticeContainer);
@@ -197,14 +192,12 @@ private static void logFieldCacheStats(
     }
   }
 
-  public static GtfsTableContainer loadMissingFile(
-      GtfsTableDescriptor tableDescriptor,
+  public static GtfsContainer loadMissingFile(
+      GtfsDescriptor tableDescriptor,
       ValidatorProvider validatorProvider,
       NoticeContainer noticeContainer) {
     String gtfsFilename = tableDescriptor.gtfsFilename();
-    GtfsTableContainer table =
-        tableDescriptor.createContainerForInvalidStatus(
-            GtfsTableContainer.TableStatus.MISSING_FILE);
+    GtfsContainer table = tableDescriptor.createContainerForInvalidStatus(TableStatus.MISSING_FILE);
     if (tableDescriptor.isRecommended()) {
       noticeContainer.addValidationNotice(new MissingRecommendedFileNotice(gtfsFilename));
     }

core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsContainer.java

@@ -0,0 +1,50 @@
+package org.mobilitydata.gtfsvalidator.table;
+
+import java.util.List;
+import java.util.Optional;
+
+public abstract class GtfsContainer<T extends GtfsEntity, D extends GtfsDescriptor> {
+
+  private final D descriptor;
+  private final TableStatus tableStatus;
+
+  public GtfsContainer(D descriptor, TableStatus tableStatus) {
+    this.tableStatus = tableStatus;
+    this.descriptor = descriptor;
+  }
+
+  public TableStatus getTableStatus() {
+    return tableStatus;
+  }
+
+  public D getDescriptor() {
+    return descriptor;
+  }
+
+  public abstract Class<T> getEntityClass();
+
+  public int entityCount() {
+    return getEntities().size();
+  }
+
+  public abstract List<T> getEntities();
+
+  public abstract String gtfsFilename();
+
+  public abstract Optional<T> byTranslationKey(String recordId, String recordSubId);
+
+  public boolean isMissingFile() {
+    return tableStatus == TableStatus.MISSING_FILE;
+  }
+
+  public boolean isParsedSuccessfully() {
+    switch (tableStatus) {
+      case PARSABLE_HEADERS_AND_ROWS:
+        return true;
+      case MISSING_FILE:
+        return !descriptor.isRequired();
+      default:
+        return false;
+    }
+  }
+}

core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsDescriptor.java

@@ -0,0 +1,27 @@
+package org.mobilitydata.gtfsvalidator.table;
+
+// TODO: review class name maybe GtfsFileDescriptor
+public abstract class GtfsDescriptor<T extends GtfsEntity> {
+
+  public abstract <C extends GtfsContainer> C createContainerForInvalidStatus(
+      TableStatus tableStatus);
+
+  // True if the specified file is required in a feed.
+  private boolean required;
+
+  private TableStatus tableStatus;
+
+  public abstract boolean isRecommended();
+
+  public abstract Class<T> getEntityClass();
+
+  public abstract String gtfsFilename();
+
+  public boolean isRequired() {
+    return this.required;
+  }
+
+  public void setRequired(boolean required) {
+    this.required = required;
+  }
+}

core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedContainer.java

@@ -18,20 +18,19 @@
 
 import com.google.common.base.Ascii;
 import java.util.*;
-import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer.TableStatus;
 
 /**
  * Container for a whole parsed GTFS feed with all its tables.
  *
- * <p>The tables are kept as {@code GtfsTableContainer} instances.
+ * <p>The tables are kept as {@code GtfsContainer} instances.
  */
 public class GtfsFeedContainer {
-  private final Map<String, GtfsTableContainer<?>> tables = new HashMap<>();
-  private final Map<Class<? extends GtfsTableContainer>, GtfsTableContainer<?>> tablesByClass =
+  private final Map<String, GtfsContainer<?, ?>> tables = new HashMap<>();
+  private final Map<Class<? extends GtfsContainer>, GtfsContainer<?, ?>> tablesByClass =
       new HashMap<>();
 
-  public GtfsFeedContainer(List<GtfsTableContainer<?>> tableContainerList) {
-    for (GtfsTableContainer<?> table : tableContainerList) {
+  public GtfsFeedContainer(List<GtfsContainer<?, ?>> tableContainerList) {
+    for (GtfsContainer<?, ?> table : tableContainerList) {
       tables.put(table.gtfsFilename(), table);
       tablesByClass.put(table.getClass(), table);
     }
@@ -49,11 +48,12 @@ public GtfsFeedContainer(List<GtfsTableContainer<?>> tableContainerList) {
    * @param filename file name, including ".txt" extension
    * @return GTFS table or empty if the table is not supported by schema
    */
-  public Optional<GtfsTableContainer<?>> getTableForFilename(String filename) {
-    return Optional.ofNullable(tables.getOrDefault(Ascii.toLowerCase(filename), null));
+  public <T extends GtfsContainer<?, ?>> Optional<T> getTableForFilename(String filename) {
+    return (Optional<T>)
+        Optional.ofNullable(tables.getOrDefault(Ascii.toLowerCase(filename), null));
   }
 
-  public <T extends GtfsTableContainer<?>> T getTable(Class<T> clazz) {
+  public <T extends GtfsContainer<?, ?>> T getTable(Class<T> clazz) {
     return (T) tablesByClass.get(clazz);
   }
 
@@ -65,21 +65,21 @@ public <T extends GtfsTableContainer<?>> T getTable(Class<T> clazz) {
    * @return true if all files were successfully parsed, false otherwise
    */
   public boolean isParsedSuccessfully() {
-    for (GtfsTableContainer<?> table : tables.values()) {
+    for (GtfsContainer<?, ?> table : tables.values()) {
       if (!table.isParsedSuccessfully()) {
         return false;
       }
     }
     return true;
   }
 
-  public Collection<GtfsTableContainer<?>> getTables() {
+  public Collection<GtfsContainer<?, ?>> getTables() {
     return tables.values();
   }
 
   public String tableTotalsText() {
     List<String> totalList = new ArrayList<>();
-    for (GtfsTableContainer<?> table : tables.values()) {
+    for (GtfsContainer<?, ?> table : tables.values()) {
       if (table.getTableStatus() == TableStatus.MISSING_FILE
           && !table.getDescriptor().isRequired()) {
         continue;

core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java

@@ -19,6 +19,7 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.flogger.FluentLogger;
 import java.io.InputStream;
+import java.lang.reflect.Modifier;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -46,7 +47,7 @@
  */
 public class GtfsFeedLoader {
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
-  private final HashMap<String, GtfsTableDescriptor<?>> tableDescriptors = new HashMap<>();
+  private final HashMap<String, GtfsDescriptor<?>> tableDescriptors = new HashMap<>();
   private int numThreads = 1;
 
   /**
@@ -56,12 +57,15 @@ public class GtfsFeedLoader {
   private final List<Class<? extends FileValidator>> multiFileValidatorsWithParsingErrors =
       new ArrayList<>();
 
-  public GtfsFeedLoader(
-      ImmutableList<Class<? extends GtfsTableDescriptor<?>>> tableDescriptorClasses) {
-    for (Class<? extends GtfsTableDescriptor<?>> clazz : tableDescriptorClasses) {
-      GtfsTableDescriptor<?> descriptor;
+  public GtfsFeedLoader(ImmutableList<Class<? extends GtfsDescriptor<?>>> tableDescriptorClasses) {
+    for (Class<? extends GtfsDescriptor<?>> clazz : tableDescriptorClasses) {
+      GtfsDescriptor<?> descriptor;
       try {
-        descriptor = clazz.asSubclass(GtfsTableDescriptor.class).getConstructor().newInstance();
+        //        Skipping abstract classes. Example: GtfsTableDescriptor.
+        if (Modifier.isAbstract(clazz.getModifiers())) {
+          continue;
+        }
+        descriptor = clazz.asSubclass(GtfsDescriptor.class).getConstructor().newInstance();
       } catch (ReflectiveOperationException e) {
         logger.atSevere().withCause(e).log(
             "Possible bug in GTFS annotation processor: expected a constructor without parameters"
@@ -73,7 +77,7 @@ public GtfsFeedLoader(
     }
   }
 
-  public Collection<GtfsTableDescriptor<?>> getTableDescriptors() {
+  public Collection<GtfsDescriptor<?>> getTableDescriptors() {
     return Collections.unmodifiableCollection(tableDescriptors.values());
   }
 
@@ -100,19 +104,36 @@ public GtfsFeedContainer loadAndValidate(
     Map<String, GtfsTableDescriptor<?>> remainingDescriptors =
         (Map<String, GtfsTableDescriptor<?>>) tableDescriptors.clone();
     for (String filename : gtfsInput.getFilenames()) {
-      GtfsTableDescriptor<?> tableDescriptor = remainingDescriptors.remove(filename.toLowerCase());
+      GtfsDescriptor<?> tableDescriptor = remainingDescriptors.remove(filename.toLowerCase());
       if (tableDescriptor == null) {
         noticeContainer.addValidationNotice(new UnknownFileNotice(filename));
       } else {
         loaderCallables.add(
             () -> {
               NoticeContainer loaderNotices = new NoticeContainer();
-              GtfsTableContainer<?> tableContainer;
+              GtfsContainer<?, ?> tableContainer;
               try (InputStream inputStream = gtfsInput.getFile(filename)) {
                 try {
-                  tableContainer =
-                      AnyTableLoader.load(
-                          tableDescriptor, validatorProvider, inputStream, loaderNotices);
+                  if (tableDescriptor instanceof GtfsTableDescriptor) {
+                    tableContainer =
+                        AnyTableLoader.load(
+                            (GtfsTableDescriptor) tableDescriptor,
+                            validatorProvider,
+                            inputStream,
+                            loaderNotices);
+                  } else if (tableDescriptor instanceof GtfsGeojsonFeatureDescriptor) {
+                    tableContainer =
+                        JsonFileLoader.load(
+                            (GtfsGeojsonFeatureDescriptor) tableDescriptor,
+                            validatorProvider,
+                            inputStream,
+                            loaderNotices);
+                  } else {
+                    logger.atSevere().log(
+                        "Runtime exception table descriptor not supported: %s",
+                        tableDescriptor.getClass().getName());
+                    throw new RuntimeException("Table descriptor is not a supported type");
+                  }
                 } catch (RuntimeException e) {
                   // This handler should prevent ExecutionException for
                   // this thread. We catch an exception here for storing
@@ -130,9 +151,9 @@ public GtfsFeedContainer loadAndValidate(
             });
       }
     }
-    ArrayList<GtfsTableContainer<?>> tableContainers = new ArrayList<>();
+    ArrayList<GtfsContainer<?, ?>> tableContainers = new ArrayList<>();
     tableContainers.ensureCapacity(tableDescriptors.size());
-    for (GtfsTableDescriptor<?> tableDescriptor : remainingDescriptors.values()) {
+    for (GtfsDescriptor<?> tableDescriptor : remainingDescriptors.values()) {
       tableContainers.add(
           AnyTableLoader.loadMissingFile(tableDescriptor, validatorProvider, noticeContainer));
     }
@@ -186,11 +207,10 @@ private static void addThreadExecutionError(
   }
 
   static class TableAndNoticeContainers {
-    final GtfsTableContainer tableContainer;
+    final GtfsContainer tableContainer;
     final NoticeContainer noticeContainer;
 
-    public TableAndNoticeContainers(
-        GtfsTableContainer tableContainer, NoticeContainer noticeContainer) {
+    public TableAndNoticeContainers(GtfsContainer tableContainer, NoticeContainer noticeContainer) {
       this.tableContainer = tableContainer;
       this.noticeContainer = noticeContainer;
     }