Skip to content

Commit

Permalink
modelhelper code compiling
Browse files Browse the repository at this point in the history
  • Loading branch information
@sonalgoyal
sonalgoyal committed May 11, 2024
1 parent 2620ba1 commit 29ec02d
Show file tree
Hide file tree
Showing 37 changed files with 370 additions and 297 deletions.
102 changes: 2 additions & 100 deletions common/client/src/main/java/zingg/common/client/Arguments.java
View file
Expand Up @@ -352,105 +352,7 @@ public void setZinggDir(String zinggDir) {
* @return the path for internal Zingg usage
*/

@Override
@JsonIgnore
public String getZinggBaseModelDir(){
return zinggDir + "/" + modelId;
}
@Override
@JsonIgnore
public String getZinggModelDir() {
return getZinggBaseModelDir() + "/model";
}

@Override
@JsonIgnore
public String getZinggDocDir() {
return getZinggBaseModelDir() + "/docs/";
}

@Override
@JsonIgnore
public String getZinggModelDocFile() {
return getZinggDocDir() + "/model.html";
}

@Override
@JsonIgnore
public String getZinggDataDocFile() {
return getZinggDocDir() + "/data.html";
}

/**
* Location for internal Zingg use.
*
* @return the path for internal Zingg usage
*/
@Override
@JsonIgnore
public String getZinggBaseTrainingDataDir() {
return getZinggBaseModelDir() + "/trainingData/";
}



/**
* Location for internal Zingg use.
*
* @return the path for internal Zingg usage
*/
@Override
@JsonIgnore
public String getZinggTrainingDataUnmarkedDir() {
return this.getZinggBaseTrainingDataDir() + "/unmarked/";
}

/**
* Location for internal Zingg use.
*
* @return the path for internal Zingg usage
*/
@Override
@JsonIgnore
public String getZinggTrainingDataMarkedDir() {
return this.getZinggBaseTrainingDataDir() + "/marked/";
}

/**
* Location for internal Zingg use.
*
* @return the path for internal Zingg usage
*/
@Override
@JsonIgnore
public String getZinggPreprocessedDataDir() {
return zinggDir + "/preprocess";
}

/**
* This is an internal block file location Not to be used directly by the
* client
*
* @return the blockFile
*/
@Override
@JsonIgnore
public String getBlockFile() {
return getZinggModelDir() + "/block/zingg.block";
}

/**
* This is the internal model location Not to be used by the client
*
* @return model path
*/
@Override
@JsonIgnore
public String getModel() {
return getZinggModelDir() + "/classifier/best.model";
}



@Override
public int getJobId() {
Expand Down Expand Up @@ -531,9 +433,9 @@ public String[] getPipeNames() {
}

@Override
@JsonIgnore
@JsonIgnore
public String getStopWordsDir() {
return getZinggBaseModelDir() + "/stopWords/";
return getZinggDir() + "/" + getModelId() + "/stopWords/";
}

}
Expand Down
18 changes: 8 additions & 10 deletions common/client/src/main/java/zingg/common/client/ArgumentsUtil.java
View file
Expand Up @@ -17,21 +17,19 @@
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.module.scala.DefaultScalaModule;

public class ArgumentsUtil {
public class ArgumentsUtil<A extends IZArgs> {

protected Class<? extends IZArgs> argsClass;
protected Class<A> argsClass;
private static final String ENV_VAR_MARKER_START = "$";
private static final String ENV_VAR_MARKER_END = "$";
private static final String ESC = "\\";
private static final String PATTERN_ENV_VAR = ESC + ENV_VAR_MARKER_START + "(.+?)" + ESC + ENV_VAR_MARKER_END;
public static final Log LOG = LogFactory.getLog(ArgumentsUtil.class);


public ArgumentsUtil() {
this(Arguments.class);
}


public ArgumentsUtil(Class<? extends IZArgs> argsClass) {
public ArgumentsUtil(Class<A> argsClass) {
this.argsClass = argsClass;
}

Expand All @@ -58,7 +56,7 @@ public IZArgs createArgumentsFromJSON(String filePath)
* @throws ZinggClientException
* in case of invlaid/wrong json/file not found
*/
public IZArgs createArgumentsFromJSON(String filePath, String phase)
public A createArgumentsFromJSON(String filePath, String phase)
throws ZinggClientException {
try {
ObjectMapper mapper = new ObjectMapper();
Expand All @@ -69,7 +67,7 @@ public IZArgs createArgumentsFromJSON(String filePath, String phase)
module.addDeserializer(List<MatchType>.class, new FieldDefinition.MatchTypeDeserializer());
mapper.registerModule(module);
*/
IZArgs args = mapper.readValue(new File(filePath), argsClass);
A args = mapper.readValue(new File(filePath), argsClass);
LOG.warn("phase is " + phase);
//checkValid(args, phase);
return args;
Expand Down Expand Up @@ -117,13 +115,13 @@ else if (!phase.equalsIgnoreCase("WEB")){
}
*/

public IZArgs createArgumentsFromJSONString(String data, String phase)
public A createArgumentsFromJSONString(String data, String phase)
throws ZinggClientException {
try {
ObjectMapper mapper = new ObjectMapper();
mapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS,
true);
IZArgs args = mapper.readValue(data, argsClass);
A args = mapper.readValue(data, argsClass);
LOG.warn("phase is " + phase);
//checkValid(args, phase);
return args;
Expand Down
12 changes: 6 additions & 6 deletions common/client/src/main/java/zingg/common/client/Client.java
View file
Expand Up @@ -26,7 +26,7 @@
public abstract class Client<S,D,R,C,T> implements Serializable {
private static final long serialVersionUID = 1L;
protected IZArgs arguments;
protected ArgumentsUtil argsUtil;
protected ArgumentsUtil<?> argsUtil;
protected IZingg<S,D,R,C> zingg;
protected ClientOptions options;
protected S session;
Expand Down Expand Up @@ -202,13 +202,13 @@ public void mainMethod(String... args) {
String phase = options.get(ClientOptions.PHASE).value.trim();
ZinggOptions.verifyPhase(phase);
if (options.get(ClientOptions.CONF).value.endsWith("json")) {
arguments = getArgsUtil().createArgumentsFromJSON(options.get(ClientOptions.CONF).value, phase);
arguments = getArgsUtil(phase).createArgumentsFromJSON(options.get(ClientOptions.CONF).value, phase);
}
else if (options.get(ClientOptions.CONF).value.endsWith("env")) {
arguments = getArgsUtil().createArgumentsFromJSONTemplate(options.get(ClientOptions.CONF).value, phase);
arguments = getArgsUtil(phase).createArgumentsFromJSONTemplate(options.get(ClientOptions.CONF).value, phase);
}
else {
arguments = getArgsUtil().createArgumentsFromJSONString(options.get(ClientOptions.CONF).value, phase);
arguments = getArgsUtil(phase).createArgumentsFromJSONString(options.get(ClientOptions.CONF).value, phase);
}

client = getClient(arguments, options);
Expand Down Expand Up @@ -298,9 +298,9 @@ public void setOptions(ClientOptions options) {
}


protected ArgumentsUtil getArgsUtil() {
protected ArgumentsUtil<?> getArgsUtil(String phase) {
if (argsUtil==null) {
argsUtil = new ArgumentsUtil();
argsUtil = new ArgumentsUtil(Arguments.class);
}
return argsUtil;
}
Expand Down
60 changes: 1 addition & 59 deletions common/client/src/main/java/zingg/common/client/IArguments.java
View file
Expand Up @@ -113,65 +113,7 @@ public Pipe[] getZinggInternal() {
*/
void setData(Pipe[] dataFile) throws ZinggClientException;

/**
* Location for internal Zingg use.
*
* @return the path for internal Zingg usage
*/

String getZinggBaseModelDir();

String getZinggModelDir();

String getZinggDocDir();

String getZinggModelDocFile();

String getZinggDataDocFile();

/**
* Location for internal Zingg use.
*
* @return the path for internal Zingg usage
*/
String getZinggBaseTrainingDataDir();

/**
* Location for internal Zingg use.
*
* @return the path for internal Zingg usage
*/
String getZinggTrainingDataUnmarkedDir();

/**
* Location for internal Zingg use.
*
* @return the path for internal Zingg usage
*/
String getZinggTrainingDataMarkedDir();

/**
* Location for internal Zingg use.
*
* @return the path for internal Zingg usage
*/
String getZinggPreprocessedDataDir();

/**
* This is an internal block file location Not to be used directly by the
* client
*
* @return the blockFile
*/
String getBlockFile();

/**
* This is the internal model location Not to be used by the client
*
* @return model path
*/
String getModel();


int getJobId();

void setJobId(int jobId);
Expand Down
5 changes: 3 additions & 2 deletions common/client/src/main/java/zingg/common/client/util/DSUtil.java
View file
Expand Up @@ -3,6 +3,7 @@

import zingg.common.client.FieldDefinition;
import zingg.common.client.IArguments;
import zingg.common.client.IZArgs;
import zingg.common.client.MatchType;
import zingg.common.client.ZFrame;
import zingg.common.client.ZinggClientException;
Expand Down Expand Up @@ -246,8 +247,8 @@ public ZFrame<D, R, C> dropDuplicates(ZFrame<D, R, C> a, IArguments args) {
return a;
}

public ZFrame<D, R, C> getTraining(PipeUtilBase<S, D, R, C> pipeUtil, IArguments args) {
return getTraining(pipeUtil, args, pipeUtil.getTrainingDataMarkedPipe(args));
public ZFrame<D, R, C> getTraining(PipeUtilBase<S, D, R, C> pipeUtil, IArguments args, IModelHelper modelHelper) {
return getTraining(pipeUtil, args, modelHelper.getTrainingDataMarkedPipe(args));
}

private ZFrame<D, R, C> getTraining(PipeUtilBase<S, D, R, C> pipeUtil, IArguments args, Pipe<D,R,C> p) {
Expand Down
81 changes: 81 additions & 0 deletions common/client/src/main/java/zingg/common/client/util/IModelHelper.java
View file
@@ -0,0 +1,81 @@
package zingg.common.client.util;

import zingg.common.client.pipe.*;

import zingg.common.client.IZArgs;

public interface IModelHelper<D,R,C> {

/**
* Location for internal Zingg use.
*
* @return the path for internal Zingg usage
*/

String getZinggBaseModelDir(IZArgs args);

String getZinggModelDir(IZArgs args);

String getZinggDocDir(IZArgs args);

String getZinggModelDocFile(IZArgs args);

String getZinggDataDocFile(IZArgs args);

/**
* Location for internal Zingg use.
*
* @return the path for internal Zingg usage
*/
String getZinggBaseTrainingDataDir(IZArgs args);

/**
* Location for internal Zingg use.
*
* @return the path for internal Zingg usage
*/
String getZinggTrainingDataUnmarkedDir(IZArgs args);

/**
* Location for internal Zingg use.
*
* @return the path for internal Zingg usage
*/
String getZinggTrainingDataMarkedDir(IZArgs args);

/**
* Location for internal Zingg use.
*
* @return the path for internal Zingg usage
*/
String getZinggPreprocessedDataDir(IZArgs args);

/**
* This is an internal block file location Not to be used directly by the
* client
*
* @return the blockFile
*/
String getBlockFile(IZArgs args);

/**
* This is the internal model location Not to be used by the client
*
* @return model path
*/
String getModel(IZArgs args);

public Pipe<D, R, C> getTrainingDataUnmarkedPipe(IZArgs args);

public Pipe<D, R, C> getTrainingDataMarkedPipe(IZArgs args);

public Pipe<D, R, C> getModelDocumentationPipe(IZArgs args);

public Pipe<D, R, C> getBlockingTreePipe(IZArgs args);


public Pipe<D,R,C> getStopWordsPipe(String fileName) ;



}

0 comments on commit 29ec02d

Please sign in to comment.