Merge pull request #116 from batfish/unstable

Unstable
batfish · Apr 20, 2017 · 706f3fa · 706f3fa
2 parents f5c2fc6 + 2e5d411
commit 706f3fa
Show file tree

Hide file tree

Showing 236 changed files with 8,077 additions and 6,012 deletions.
diff --git a/.gitignore b/.gitignore
@@ -2,3 +2,4 @@
 /client
 /unit-tests
 .DS_Store
+*.testout
diff --git a/projects/batfish-client/src/org/batfish/client/config/client.properties b/projects/batfish-client/src/org/batfish/client/config/client.properties
@@ -2,3 +2,7 @@
 #apikey = 0000000000000000000000000000000
 
 #coordinatorhost = www.batfish.org
+
+#directories storing extra plugins
+plugindirs = /etc/batfish/plugins
+
diff --git a/projects/batfish-common-protocol/src/org/batfish/common/BfConsts.java b/projects/batfish-common-protocol/src/org/batfish/common/BfConsts.java
@@ -76,13 +76,15 @@ public enum TaskStatus {
    public static final String RELPATH_CONFIG_FILE_NAME_COORDINATOR = "coordinator.properties";
    public static final String RELPATH_CONFIGURATIONS_DIR = "configs";
    public static final String RELPATH_CONVERT_ANSWER_PATH = "convert_answer";
+   public static final String RELPATH_DATA_PLANE = "dp";
    public static final String RELPATH_DATA_PLANE_ANSWER_PATH = "dp_answer";
-   public static final String RELPATH_DATA_PLANE_DIR = "dp";
    public static final String RELPATH_DEFAULT_ENVIRONMENT_NAME = "env_default";
    public static final String RELPATH_DIFF = "diff";
    public static final String RELPATH_EDGE_BLACKLIST_FILE = "edge_blacklist";
    public static final String RELPATH_ENV_DIR = "env";
    public static final String RELPATH_ENV_NODE_SET = "env-node-set";
+   public static final String RELPATH_ENVIRONMENT_ROUTING_TABLES = "rt";
+   public static final String RELPATH_ENVIRONMENT_ROUTING_TABLES_ANSWER = "rt_answer";
    public static final String RELPATH_ENVIRONMENTS_DIR = "environments";
    public static final String RELPATH_EXTERNAL_BGP_ANNOUNCEMENTS = "external_bgp_announcements.json";
    public static final String RELPATH_FAILURE_QUERY_PREFIX = "failure-query";
@@ -96,6 +98,7 @@ public enum TaskStatus {
    public static final String RELPATH_QUERIES_DIR = "queries";
    public static final String RELPATH_QUESTION_FILE = "question.json";
    public static final String RELPATH_QUESTIONS_DIR = "questions";
+   public static final String RELPATH_SERIALIZED_ENVIRONMENT_ROUTING_TABLES = "rt_processed";
    public static final String RELPATH_TEST_RIG_DIR = "testrig";
    public static final String RELPATH_TOPOLOGY_FILE = "topology";
    public static final String RELPATH_VENDOR_INDEPENDENT_CONFIG_DIR = "indep";

diff --git a/...atfish/main/PedanticBatfishException.java → ...fish/common/PedanticBatfishException.java b/...atfish/main/PedanticBatfishException.java → ...fish/common/PedanticBatfishException.java
@@ -1,6 +1,4 @@
-package org.batfish.main;
-
-import org.batfish.common.BatfishException;
+package org.batfish.common;
 
 public class PedanticBatfishException extends BatfishException {
 

diff --git a/...batfish/main/RedFlagBatfishException.java → ...tfish/common/RedFlagBatfishException.java b/...batfish/main/RedFlagBatfishException.java → ...tfish/common/RedFlagBatfishException.java
@@ -1,6 +1,4 @@
-package org.batfish.main;
-
-import org.batfish.common.BatfishException;
+package org.batfish.common;
 
 public class RedFlagBatfishException extends BatfishException {
 

diff --git a/...h/main/UnimplementedBatfishException.java → ...common/UnimplementedBatfishException.java b/...h/main/UnimplementedBatfishException.java → ...common/UnimplementedBatfishException.java
@@ -1,6 +1,4 @@
-package org.batfish.main;
-
-import org.batfish.common.BatfishException;
+package org.batfish.common;
 
 public class UnimplementedBatfishException extends BatfishException {
 

diff --git a/projects/batfish-common-protocol/src/org/batfish/common/Version.java b/projects/batfish-common-protocol/src/org/batfish/common/Version.java
@@ -8,7 +8,7 @@ public final class Version {
 
    private static final String UNKNOWN_VERSION = "0.0.0";
 
-   private static final String VERSION = "0.16.0";
+   private static final String VERSION = "0.17.0";
 
    public static void checkCompatibleVersion(String myName, String otherName,
          String otherVersion) {

diff --git a/projects/batfish-common-protocol/src/org/batfish/common/Warnings.java b/projects/batfish-common-protocol/src/org/batfish/common/Warnings.java
@@ -3,10 +3,15 @@
 import java.io.IOException;
 import java.io.Serializable;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map.Entry;
 
+import org.antlr.v4.runtime.ParserRuleContext;
+import org.batfish.grammar.BatfishCombinedParser;
+import org.batfish.grammar.ParseTreePrettyPrinter;
+
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.core.JsonGenerator;
 import com.fasterxml.jackson.core.JsonParser;
@@ -111,6 +116,8 @@ public void serialize(Warnings value, JsonGenerator jgen,
 
    }
 
+   private static final String MISCELLANEOUS = "MISCELLANEOUS";
+
    private static final String PEDANTIC_VAR = "Pedantic complaints";
 
    private static final String RED_FLAGS_VAR = "Red flags";
@@ -122,10 +129,24 @@ public void serialize(Warnings value, JsonGenerator jgen,
 
    private static final String UNIMPLEMENTED_VAR = "Unimplemented features";
 
+   private transient boolean _pedanticAsError;
+
+   private transient boolean _pedanticRecord;
+
    protected final List<Warning> _pedanticWarnings;
 
+   private transient boolean _printParseTree;
+
+   private transient boolean _redFlagAsError;
+
+   private transient boolean _redFlagRecord;
+
    protected final List<Warning> _redFlagWarnings;
 
+   private transient boolean _unimplementedAsError;
+
+   private transient boolean _unimplementedRecord;
+
    protected final List<Warning> _unimplementedWarnings;
 
    public Warnings() {
@@ -135,6 +156,20 @@ public Warnings() {
 
    }
 
+   public Warnings(boolean pedanticAsError, boolean pedanticRecord,
+         boolean redFlagAsError, boolean redFlagRecord,
+         boolean unimplementedAsError, boolean unimplementedRecord,
+         boolean printParseTree) {
+      this();
+      _pedanticAsError = pedanticAsError;
+      _pedanticRecord = pedanticRecord;
+      _printParseTree = printParseTree;
+      _redFlagAsError = redFlagAsError;
+      _redFlagRecord = redFlagRecord;
+      _unimplementedAsError = unimplementedAsError;
+      _unimplementedRecord = unimplementedRecord;
+   }
+
    public List<Warning> getPedanticWarnings() {
       return _pedanticWarnings;
    }
@@ -153,4 +188,86 @@ public boolean isEmpty() {
             && _unimplementedWarnings.isEmpty();
    }
 
+   public void pedantic(String msg) {
+      pedantic(msg, MISCELLANEOUS);
+   }
+
+   public void pedantic(String msg, String tag) {
+      if (_pedanticAsError) {
+         throw new PedanticBatfishException(msg);
+      }
+      else if (_pedanticRecord) {
+         _pedanticWarnings.add(new Warning(msg, tag));
+      }
+   }
+
+   public void redFlag(String msg) {
+      redFlag(msg, MISCELLANEOUS);
+   }
+
+   public void redFlag(String msg, String tag) {
+      if (_redFlagAsError) {
+         throw new RedFlagBatfishException(msg);
+      }
+      else if (_redFlagRecord) {
+         _redFlagWarnings.add(new Warning(msg, tag));
+      }
+   }
+
+   public void todo(ParserRuleContext ctx, String feature,
+         BatfishCombinedParser<?, ?> parser, String text) {
+      if (!_unimplementedRecord && !_unimplementedAsError) {
+         return;
+      }
+      String prefix = "WARNING: UNIMPLEMENTED: "
+            + (_unimplementedWarnings.size() + 1) + ": ";
+      StringBuilder sb = new StringBuilder();
+      List<String> ruleNames = Arrays.asList(parser.getParser().getRuleNames());
+      String ruleStack = ctx.toString(ruleNames);
+      sb.append(
+            prefix + "Missing implementation for top (leftmost) parser rule in stack: '"
+                  + ruleStack + "'.\n");
+      sb.append(prefix + "Unimplemented feature: " + feature + "\n");
+      sb.append(prefix + "Rule context follows:\n");
+      int start = ctx.start.getStartIndex();
+      int startLine = ctx.start.getLine();
+      int end = ctx.stop.getStopIndex();
+      String ruleText = text.substring(start, end + 1);
+      String[] ruleTextLines = ruleText.split("\\n");
+      for (int line = startLine, i = 0; i < ruleTextLines.length; line++, i++) {
+         String contextPrefix = prefix + " line " + line + ": ";
+         sb.append(contextPrefix + ruleTextLines[i] + "\n");
+      }
+      if (_printParseTree) {
+         sb.append(prefix + "Parse tree follows:\n");
+         String parseTreePrefix = prefix + "PARSE TREE: ";
+         String parseTreeText = ParseTreePrettyPrinter.print(ctx, parser);
+         String[] parseTreeLines = parseTreeText.split("\n");
+         for (String parseTreeLine : parseTreeLines) {
+            sb.append(parseTreePrefix + parseTreeLine + "\n");
+         }
+      }
+      String warning = sb.toString();
+      if (_unimplementedAsError) {
+         throw new UnimplementedBatfishException(warning);
+      }
+      else {
+         _unimplementedWarnings
+               .add(new Warning(sb.toString(), "UNIMPLEMENTED"));
+      }
+   }
+
+   public void unimplemented(String msg) {
+      unimplemented(msg, MISCELLANEOUS);
+   }
+
+   public void unimplemented(String msg, String tag) {
+      if (_unimplementedAsError) {
+         throw new UnimplementedBatfishException(msg);
+      }
+      else if (_unimplementedRecord) {
+         _unimplementedWarnings.add(new Warning(msg, tag));
+      }
+   }
+
 }
diff --git a/...batfish-common-protocol/src/org/batfish/common/plugin/ExternalBgpAdvertisementPlugin.java b/...batfish-common-protocol/src/org/batfish/common/plugin/ExternalBgpAdvertisementPlugin.java
@@ -0,0 +1,35 @@
+package org.batfish.common.plugin;
+
+import org.batfish.datamodel.collections.AdvertisementSet;
+
+public abstract class ExternalBgpAdvertisementPlugin extends BatfishPlugin
+      implements IExternalBgpAdvertisementPlugin,
+      Comparable<ExternalBgpAdvertisementPlugin> {
+
+   @Override
+   protected final void batfishPluginInitialize() {
+      _batfish.registerExternalBgpAdvertisementPlugin(this);
+      externalBgpAdvertisementPluginInitialize();
+   }
+
+   @Override
+   public int compareTo(ExternalBgpAdvertisementPlugin o) {
+      return getClass().getCanonicalName()
+            .compareTo(o.getClass().getCanonicalName());
+   }
+
+   @Override
+   public boolean equals(Object obj) {
+      return getClass().equals(obj.getClass());
+   }
+
+   protected abstract void externalBgpAdvertisementPluginInitialize();
+
+   @Override
+   public int hashCode() {
+      return getClass().hashCode();
+   }
+
+   public abstract AdvertisementSet loadExternalBgpAdvertisements();
+
+}
diff --git a/projects/batfish-common-protocol/src/org/batfish/common/plugin/IBatfish.java b/projects/batfish-common-protocol/src/org/batfish/common/plugin/IBatfish.java
@@ -2,6 +2,7 @@
 
 import java.util.Map;
 import java.util.Set;
+import java.util.SortedMap;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.BiFunction;
 
@@ -19,13 +20,15 @@
 import org.batfish.datamodel.answers.ConvertConfigurationAnswerElement;
 import org.batfish.datamodel.answers.DataPlaneAnswerElement;
 import org.batfish.datamodel.answers.InitInfoAnswerElement;
+import org.batfish.datamodel.answers.ParseEnvironmentRoutingTablesAnswerElement;
 import org.batfish.datamodel.answers.ParseVendorConfigurationAnswerElement;
 import org.batfish.datamodel.assertion.AssertionAst;
 import org.batfish.datamodel.collections.AdvertisementSet;
 import org.batfish.datamodel.collections.InterfaceSet;
 import org.batfish.datamodel.collections.NamedStructureEquivalenceSets;
 import org.batfish.datamodel.collections.NodeInterfacePair;
 import org.batfish.datamodel.collections.NodeSet;
+import org.batfish.datamodel.collections.RoutesByVrf;
 import org.batfish.datamodel.questions.Question;
 import org.batfish.grammar.GrammarSettings;
 
@@ -71,7 +74,7 @@ AnswerElement createEnvironment(String environmentName,
    void initBgpOriginationSpaceExplicit(
          Map<String, Configuration> configurations);
 
-   InitInfoAnswerElement initInfo(boolean summary);
+   InitInfoAnswerElement initInfo(boolean summary, boolean environmentRoutes);
 
    void initRemoteBgpNeighbors(Map<String, Configuration> configurations,
          Map<Ip, Set<String>> ipOwners);
@@ -83,12 +86,16 @@ void initRemoteOspfNeighbors(Map<String, Configuration> configurations,
 
    void initRoutes(Map<String, Configuration> configurations);
 
-   Map<String, Configuration> loadConfigurations();
+   SortedMap<String, Configuration> loadConfigurations();
 
    ConvertConfigurationAnswerElement loadConvertConfigurationAnswerElement();
 
    DataPlane loadDataPlane();
 
+   SortedMap<String, RoutesByVrf> loadEnvironmentRoutingTables();
+
+   ParseEnvironmentRoutingTablesAnswerElement loadParseEnvironmentRoutingTablesAnswerElement();
+
    ParseVendorConfigurationAnswerElement loadParseVendorConfigurationAnswerElement();
 
    AnswerElement multipath(HeaderSpace headerSpace);
@@ -112,11 +119,16 @@ AdvertisementSet processExternalBgpAnnouncements(
 
    void pushDeltaEnvironment();
 
+   String readExternalBgpAnnouncementsFile();
+
    AnswerElement reducedReachability(HeaderSpace headerSpace);
 
    void registerAnswerer(String questionName, String questionClassName,
          BiFunction<Question, IBatfish, Answerer> answererCreator);
 
+   void registerExternalBgpAdvertisementPlugin(
+         ExternalBgpAdvertisementPlugin externalBgpAdvertisementPlugin);
+
    void resetTimer();
 
    void setDataPlanePlugin(DataPlanePlugin dataPlanePlugin);

diff --git a/...atfish-common-protocol/src/org/batfish/common/plugin/IExternalBgpAdvertisementPlugin.java b/...atfish-common-protocol/src/org/batfish/common/plugin/IExternalBgpAdvertisementPlugin.java
@@ -0,0 +1,5 @@
+package org.batfish.common.plugin;
+
+public interface IExternalBgpAdvertisementPlugin extends IBatfishPlugin {
+
+}
diff --git a/projects/batfish-common-protocol/src/org/batfish/common/plugin/PluginConsumer.java b/projects/batfish-common-protocol/src/org/batfish/common/plugin/PluginConsumer.java
@@ -204,19 +204,21 @@ private void loadPluginJar(Path path) {
 
    protected final void loadPlugins() {
       for (Path pluginDir : _pluginDirs) {
-         try {
-            Files.walkFileTree(pluginDir, new SimpleFileVisitor<Path>() {
-               @Override
-               public FileVisitResult visitFile(Path path,
-                     BasicFileAttributes attrs) throws IOException {
-                  loadPluginJar(path);
-                  return FileVisitResult.CONTINUE;
-               }
-            });
-         }
-         catch (IOException e) {
-            throw new BatfishException("Error walking through plugin dir: '"
-                  + pluginDir.toString() + "'", e);
+         if (Files.exists(pluginDir)) {
+            try {
+               Files.walkFileTree(pluginDir, new SimpleFileVisitor<Path>() {
+                  @Override
+                  public FileVisitResult visitFile(Path path,
+                        BasicFileAttributes attrs) throws IOException {
+                     loadPluginJar(path);
+                     return FileVisitResult.CONTINUE;
+                  }
+               });
+            }
+            catch (IOException e) {
+               throw new BatfishException("Error walking through plugin dir: '"
+                     + pluginDir.toString() + "'", e);
+            }
          }
       }
    }

diff --git a/projects/batfish-common-protocol/src/org/batfish/common/util/CommonUtil.java b/projects/batfish-common-protocol/src/org/batfish/common/util/CommonUtil.java
@@ -8,6 +8,7 @@
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.security.MessageDigest;
@@ -114,6 +115,8 @@ public static void delete(Path path) {
       try {
          Files.delete(path);
       }
+      catch (NoSuchFileException e) {
+      }
       catch (IOException e) {
          throw new BatfishException("Failed to delete file: " + path, e);
       }