Skip to content
This repository has been archived by the owner on Jul 7, 2020. It is now read-only.

changed the file writing mode to avoid an empty minion.state wh… #268

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

changed the file writing mode to avoid an empty minion.state wh… #268

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
3 changes: 1 addition & 2 deletions hydra-main/src/main/java/com/addthis/hydra/minion/CommandTaskDeleteRunner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ public void run() {
task.setDeleted(true);
minion.tasks.remove(task.getJobKey().toString());
log.warn("[task.delete] {} terminated={}", task.getJobKey(), terminated);
minion.writeState();
minion.writeState(true);
}
File taskDirFile = new File(minion.rootDir + "/" + delete.getJobUuid() + (delete.getNodeID() != null ? "/" + delete.getNodeID() : ""));
if (taskDirFile.exists() && taskDirFile.isDirectory()) {
Expand All @@ -52,6 +52,5 @@ public void run() {
} finally {
minion.minionStateLock.unlock();
}

}
}
2 changes: 1 addition & 1 deletion hydra-main/src/main/java/com/addthis/hydra/minion/CommandTaskRevertRunner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,6 @@ public void run() {
log.warn("[task.revert] " + task.getJobKey() + " completed in " + (System.currentTimeMillis() - time) + "ms.");
}
}
minion.writeState();
minion.writeState(true);
}
}
2 changes: 1 addition & 1 deletion hydra-main/src/main/java/com/addthis/hydra/minion/CommandTaskStopRunner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ public void run() {
task.sendEndStatus(task.findLastJobStatus());
}
}
minion.writeState();
minion.writeState(true);
}

}
14 changes: 9 additions & 5 deletions hydra-main/src/main/java/com/addthis/hydra/minion/JobTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,10 @@
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.function.Function;

import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.function.Function;

import com.addthis.basis.util.LessBytes;
import com.addthis.basis.util.LessFiles;
Expand All @@ -35,7 +35,11 @@
import com.addthis.codec.annotations.FieldConfig;
import com.addthis.codec.codables.Codable;
import com.addthis.codec.json.CodecJSON;
import com.addthis.hydra.job.*;
import com.addthis.hydra.job.BackupWorkItem;
import com.addthis.hydra.job.JobTaskErrorCode;
import com.addthis.hydra.job.JobTaskState;
import com.addthis.hydra.job.ReplicateWorkItem;
import com.addthis.hydra.job.RunTaskWorkItem;
import com.addthis.hydra.job.backup.DailyBackup;
import com.addthis.hydra.job.backup.GoldBackup;
import com.addthis.hydra.job.backup.HourlyBackup;
Expand Down Expand Up @@ -446,7 +450,7 @@ private List<String> assembleBackupCommandsForHost(boolean local, ReplicaTarget
}
}
}
minion.writeState();
minion.writeState(true);
return copyCommands;
}

Expand Down Expand Up @@ -1255,15 +1259,15 @@ public boolean stopWait(File[] pidFiles, boolean kill) {
return result;
}

private void resetStartTime() {
protected void resetStartTime() {
if (isRunning()) {
startTime = 0;
} else if (isReplicating()) {
replicateStartTime = 0;
} else if (isBackingUp()) {
backupStartTime = 0;
}
minion.writeState();
minion.writeState(true);
}

public File getLiveDir() {
Expand Down
14 changes: 7 additions & 7 deletions hydra-main/src/main/java/com/addthis/hydra/minion/Minion.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -278,7 +278,7 @@ private Minion(@JsonProperty("dataDir") File rootDir,
if (liveEverywhereMarkerFile.createNewFile()) {
log.info("cutover to live-everywhere tasks");
}
writeState();
writeState(false);
if (!Strings.isNullOrEmpty(queueType)) {
runner = new TaskRunner(this);
runner.start();
Expand Down Expand Up @@ -421,7 +421,7 @@ public void insertJobKickMessage(CommandTaskKick kick) {
} finally {
minionStateLock.unlock();
}
writeState();
writeState(true);
}

void kickNextJob() throws Exception {
Expand Down Expand Up @@ -462,7 +462,7 @@ void kickNextJob() throws Exception {
log.warn("[kick] exception while trying to kick {}", task.getName(), ex);
task.sendEndStatus(JobTaskErrorCode.EXIT_SCRIPT_EXEC_ERROR);
}
writeState();
writeState(true);
return;
}
} finally {
Expand Down Expand Up @@ -581,10 +581,10 @@ private boolean updateTaskMeta(String jobID, File taskRoot) throws IOException {
}
}

void writeState() {
void writeState(boolean append) {
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is append the correct thing to do? Minion state is a JSON object. If you keep appending, wont that result in invalid json in the file? Have you tested that?

minionStateLock.lock();
try {
LessFiles.write(stateFile, LessBytes.toBytes(CodecJSON.encodeString(this)), false);
LessFiles.write(stateFile, LessBytes.toBytes(CodecJSON.encodeString(this)), append);
} catch (IOException io) {
log.warn("Error writing minion state to disk: ", io);
/* assume disk failure: set diskReadOnly=true and exit */
Expand Down Expand Up @@ -797,7 +797,7 @@ JobTask createNewTask(String jobID, int node) throws ExecException {
log.info("[task.new] restore {}/{} root={}", task.id, task.node, task.taskRoot);
tasks.put(task.getJobKey().toString(), task);
task.initializeFileVariables();
writeState();
writeState(true);
return task;
}

Expand Down Expand Up @@ -842,7 +842,7 @@ public boolean getShutdown() {
minionGroupMembership.removeFromGroup("/minion/up", getUUID());
zkClient.close();
}
writeState();
writeState(true);
}
}
}
Expand Down
16 changes: 14 additions & 2 deletions hydra-main/src/main/java/com/addthis/hydra/minion/TaskRunner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,13 @@ public void stopTaskRunner() {
minion.channel.basicAck(delivery.getEnvelope().getDeliveryTag(), false);
} catch (InterruptedException ex) {
log.warn("Interrupted while processing task messages");
minion.shutdown();
try {
minion.close();
} catch (Exception e) {
log.error("Minion close throws an exception", e);
} finally {
minion.shutdown();
}
} catch (ShutdownSignalException shutdownException) {
log.warn("Received unexpected shutdown exception from rabbitMQ", shutdownException);
try {
Expand All @@ -80,7 +86,13 @@ public void stopTaskRunner() {
if (!(ex instanceof ExecException)) {
log.error("Error nacking message", ex);
}
minion.shutdown();
try {
minion.close();
} catch (Exception e) {
log.error("Minion close throws an exception", e);
} finally {
minion.shutdown();
}
}
}
}
Expand Down