CONDEC-822: Maintain changed files and their links to Jira issues in the knowledge graph as part of git fetch (#365)

* Enable to get new commits since last fetch

* Enable maintenance of links from existing code files to Jira issues: add/delete links and add/delete code files based on changes in commits from git fetch

* Update version to 2.2.8

Author: kleebaum

pom.xml

@@ -6,7 +6,7 @@
  <modelVersion>4.0.0</modelVersion>
  <groupId>de.uhd.ifi.se.decision</groupId>
  <artifactId>management.jira</artifactId>
- <version>2.2.7</version>
+ <version>2.2.8</version>
  <organization>
   <name>Software Engineering Research Group, Heidelberg University</name>
   <url>https://github.com/cures-hub</url>

src/main/java/de/uhd/ifi/se/decision/management/jira/extraction/GitClient.java

@@ -40,11 +40,8 @@
  */
 public class GitClient {
 
-	public static final long REPO_OUTDATED_AFTER = 10 * 60 * 1000; // ex. 10 minutes = 10 minutes * 60 seconds * 1000
-	// miliseconds
-	private List<GitClientForSingleRepository> gitClientsForSingleRepos;
-
 	private String projectKey;
+	private List<GitClientForSingleRepository> gitClientsForSingleRepos;
 	private static final Logger LOGGER = LoggerFactory.getLogger(GitClient.class);
 
 	/**

src/main/java/de/uhd/ifi/se/decision/management/jira/extraction/versioncontrol/GitClientForSingleRepository.java

@@ -23,7 +23,6 @@
 import org.eclipse.jgit.lib.StoredConfig;
 import org.eclipse.jgit.revwalk.RevCommit;
 import org.eclipse.jgit.revwalk.RevWalk;
-import org.eclipse.jgit.transport.FetchResult;
 import org.eclipse.jgit.transport.RemoteConfig;
 import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
 import org.eclipse.jgit.treewalk.CanonicalTreeParser;
@@ -40,7 +39,15 @@
 import de.uhd.ifi.se.decision.management.jira.persistence.singlelocations.CodeClassPersistenceManager;
 
 /**
- * Retrieves commits and code changes (diffs) from one git repository.
+ * FIXME Investigate alternative for the following decision problem:
+ * 
+ * @issue How can we assign more than one git repository to a Jira project?
+ * @decision Implement class GitClientForSingleRepository with separate git
+ *           attribute for each repository!
+ * @alternative Use git.remoteAdd() command to add new repos!
+ *
+ *              Retrieves commits and code changes (diffs) from one git
+ *              repository.
  */
 public class GitClientForSingleRepository {
 
@@ -89,7 +96,7 @@ public boolean fetchOrClone() {
 		if (gitDirectory.exists()) {
 			if (openRepository(gitDirectory)) {
 				if (!fetch()) {
-					LOGGER.error("Failed Git pull " + workingDirectory);
+					LOGGER.error("Failed Git fetch " + workingDirectory);
 					return false;
 				}
 			} else {
@@ -121,35 +128,91 @@ private boolean openRepository(File directory) {
 		return true;
 	}
 
+	/**
+	 * @issue How can we get the new commits since the last fetch? To know the new
+	 *        commits is important for trace link maintenance between Jira issues
+	 *        and changed files (this includes updating/deleting changed files and
+	 *        their links in database).
+	 * @decision Store the position of the default branch (e.g. master) before
+	 *           fetching and after fetching to determine changes (in particular new
+	 *           commits) since last fetch!
+	 * @alternative Use FetchResult and TrackingRefUpdate to determine changes (in
+	 *              particular new commits) since last fetch!
+	 * @con Might work in productive code but did not work in unit testing.
+	 * 
+	 * @return true if fetching was successful.
+	 */
 	public boolean fetch() {
-		LOGGER.info("Pulling Repository: " + repoUri);
+		LOGGER.info("Fetching Repository: " + repoUri);
 		try {
 			List<RemoteConfig> remotes = git.remoteList().call();
 			for (RemoteConfig remote : remotes) {
-				FetchResult fetchResult = git.fetch().setRemote(remote.getName()).setRefSpecs(remote.getFetchRefSpecs())
+				ObjectId oldId = getDefaultBranchPosition();
+				git.fetch().setRemote(remote.getName()).setRefSpecs(remote.getFetchRefSpecs())
 						.setRemoveDeletedRefs(true).call();
+				ObjectId newId = getDefaultBranchPosition();
+				Diff diffSinceLastFetch = getDiffSinceLastFetch(oldId, newId);
+				CodeClassPersistenceManager persistenceManager = new CodeClassPersistenceManager(projectKey);
+				persistenceManager.maintainChangedFilesInDatabase(diffSinceLastFetch);
 				LOGGER.info("Fetched branches in " + git.getRepository().getDirectory());
-				// @issue How to maintain changed files extracted from git?
-				// for (TrackingRefUpdate updateRes : fetchResult.getTrackingRefUpdates()) {
-				// String refName = updateRes.getLocalName();
-				// if (!refName.toLowerCase().contains(defaultBranchName.toLowerCase())) {
-				// continue;
-				// }
-				// Diff diffSinceLastFetch = getDiff(updateRes.getOldObjectId(),
-				// updateRes.getNewObjectId());
-				// CodeClassPersistenceManager persistenceManager = new
-				// CodeClassPersistenceManager(projectKey);
-				// persistenceManager.maintainChangedFilesInDatabase(diffSinceLastFetch);
-				// }
 			}
 		} catch (GitAPIException e) {
-			LOGGER.error("Issue occurred while pulling from a remote." + "\n\t " + e.getMessage());
+			LOGGER.error("Issue occurred while fetching from a remote." + "\n\t " + e.getMessage());
 			return false;
 		}
-		LOGGER.info("Pulled from remote in " + git.getRepository().getDirectory());
+		LOGGER.info("Fetched from remote in " + git.getRepository().getDirectory());
 		return true;
 	}
 
+	private ObjectId getDefaultBranchPosition() {
+		ObjectId objectId = null;
+		try {
+			objectId = getRepository().resolve(getDefaultBranch().getName());
+		} catch (RevisionSyntaxException | IOException | NullPointerException e) {
+		}
+		return objectId;
+	}
+
+	public Diff getDiffSinceLastFetch(ObjectId oldObjectId, ObjectId newObjectId) {
+		List<RevCommit> newCommits = getCommitsSinceLastFetch(oldObjectId, newObjectId);
+		if (newCommits.isEmpty()) {
+			return new Diff();
+		}
+		Diff diffSinceLastFetch = getDiff(newCommits.get(0), newCommits.get(newCommits.size() - 1));
+		return addCommitsToChangedFiles(diffSinceLastFetch, newCommits);
+	}
+
+	public Diff addCommitsToChangedFiles(Diff diff, List<RevCommit> commits) {
+		for (RevCommit commit : commits) {
+			List<DiffEntry> diffEntriesInCommit = getDiffEntries(commit);
+			for (DiffEntry diffEntry : diffEntriesInCommit) {
+				for (ChangedFile file : diff.getChangedFiles()) {
+					if (diffEntry.getNewPath().contains(file.getName())) {
+						file.addCommit(commit);
+					}
+				}
+			}
+		}
+		return diff;
+	}
+
+	/**
+	 * @return commits between the two git objects as a list of {@link RevCommit}s.
+	 */
+	private List<RevCommit> getCommitsSinceLastFetch(ObjectId oldObjectId, ObjectId newObjectId) {
+		if (oldObjectId == null || newObjectId == null) {
+			return new ArrayList<>();
+		}
+		List<RevCommit> newCommits = new ArrayList<>();
+		try {
+			Iterable<RevCommit> newCommitsIterable = git.log().addRange(oldObjectId, newObjectId).call();
+			newCommitsIterable.iterator().forEachRemaining(newCommits::add);
+		} catch (Exception e) {
+			LOGGER.error("Issue occurred while fetching from a remote." + "\n\t " + e.getMessage());
+		}
+		return newCommits;
+	}
+
 	private boolean cloneRepository(File directory) {
 		if (repoUri == null || repoUri.isEmpty()) {
 			return false;

src/main/java/de/uhd/ifi/se/decision/management/jira/persistence/singlelocations/CodeClassPersistenceManager.java

@@ -56,7 +56,7 @@ public boolean deleteKnowledgeElement(long id, ApplicationUser user) {
 		boolean isDeleted = false;
 		for (CodeClassInDatabase databaseEntry : ACTIVE_OBJECTS.find(CodeClassInDatabase.class,
 				Query.select().where("ID = ?", id))) {
-			GenericLinkManager.deleteLinksForElement(id, DocumentationLocation.COMMIT);
+			GenericLinkManager.deleteLinksForElement(id, documentationLocation);
 			KnowledgeGraph.getOrCreate(projectKey).removeVertex(new ChangedFile(databaseEntry));
 			isDeleted = CodeClassInDatabase.deleteElement(databaseEntry);
 		}
@@ -104,7 +104,7 @@ public KnowledgeElement getKnowledgeElement(KnowledgeElement element) {
 	public KnowledgeElement getKnowledgeElementByName(String name) {
 		KnowledgeElement element = null;
 		for (CodeClassInDatabase databaseEntry : ACTIVE_OBJECTS.find(CodeClassInDatabase.class,
-				Query.select().where("FILE_NAME = ?", name))) {
+				Query.select().where("PROJECT_KEY = ? AND FILE_NAME = ?", projectKey, name))) {
 			element = new ChangedFile(databaseEntry);
 		}
 		return element;
@@ -120,15 +120,6 @@ public List<KnowledgeElement> getKnowledgeElements() {
 		return knowledgeElements;
 	}
 
-	public List<KnowledgeElement> getKnowledgeElementsMatchingName(String fileName) {
-		List<KnowledgeElement> knowledgeElements = new ArrayList<>();
-		for (CodeClassInDatabase databaseEntry : ACTIVE_OBJECTS.find(CodeClassInDatabase.class,
-				Query.select().where("PROJECT_KEY = ? AND FILE_NAME = ?", projectKey, fileName))) {
-			knowledgeElements.add(new ChangedFile(databaseEntry));
-		}
-		return knowledgeElements;
-	}
-
 	@Override
 	public List<Link> getInwardLinks(KnowledgeElement element) {
 		return GenericLinkManager.getInwardLinks(element);
@@ -140,21 +131,22 @@ public List<Link> getOutwardLinks(KnowledgeElement element) {
 	}
 
 	@Override
-	public KnowledgeElement insertKnowledgeElement(KnowledgeElement changedFile, ApplicationUser user) {
-		if (changedFile.getDocumentationLocation() != documentationLocation) {
+	public KnowledgeElement insertKnowledgeElement(KnowledgeElement knowledgeElement, ApplicationUser user) {
+		if (knowledgeElement.getDocumentationLocation() != documentationLocation) {
 			return null;
 		}
-		ChangedFile existingElement = (ChangedFile) getKnowledgeElementByName(changedFile.getSummary());
+		ChangedFile changedFile = (ChangedFile) knowledgeElement;
+		ChangedFile existingElement = (ChangedFile) getKnowledgeElementByName(changedFile.getName());
 		if (existingElement != null) {
-			changedFile.setId(existingElement.getId());
-			createLinksToJiraIssues((ChangedFile) changedFile, user);
+			existingElement.setCommits(changedFile.getCommits());
+			createLinksToJiraIssues(existingElement, user);
 			return existingElement;
 		}
 		CodeClassInDatabase databaseEntry = ACTIVE_OBJECTS.create(CodeClassInDatabase.class);
 		setParameters(changedFile, databaseEntry);
 		databaseEntry.save();
 		ChangedFile newElement = new ChangedFile(databaseEntry);
-		newElement.setCommits(((ChangedFile) changedFile).getCommits());
+		newElement.setCommits(changedFile.getCommits());
 		createLinksToJiraIssues(newElement, user);
 
 		return newElement;
@@ -203,6 +195,9 @@ public CodeClassInDatabase findDatabaseEntry(KnowledgeElement element) {
 		return entry;
 	}
 
+	/**
+	 * @issue How to maintain changed files and links extracted from git?
+	 */
 	public void maintainChangedFilesInDatabase(Diff diff) {
 		if (diff == null || diff.getChangedFiles().isEmpty()) {
 			return;
@@ -213,35 +208,43 @@ public void maintainChangedFilesInDatabase(Diff diff) {
 		}
 	}
 
-	private void updateChangedFileInDatabase(ChangedFile changedFile) {
+	public void updateChangedFileInDatabase(ChangedFile changedFile) {
 		if (!changedFile.isJavaClass()) {
 			return;
 		}
 		DiffEntry diffEntry = changedFile.getDiffEntry();
 		switch (diffEntry.getChangeType()) {
 		case ADD:
-			insertKnowledgeElement(changedFile, null);
+			// same as modify, thus, no break after add to fall through
 		case MODIFY:
-			// same as add, thus, no break after add to fall through
 			// new links could have been added
+			handleAdd(changedFile);
 			break;
 		case RENAME:
 			handleRename(changedFile);
 			break;
 		case DELETE:
-			deleteKnowledgeElement(changedFile, null);
+			handleDelete(changedFile);
 			break;
 		default:
 			break;
 		}
 	}
 
-	private void handleRename(ChangedFile changedFile) {
-		KnowledgeElement oldFile = getKnowledgeElementByName(changedFile.getOldName());
-		deleteKnowledgeElement(oldFile, null);
+	private void handleAdd(ChangedFile changedFile) {
 		insertKnowledgeElement(changedFile, null);
 	}
 
+	private void handleDelete(ChangedFile changedFile) {
+		KnowledgeElement fileToBeDeleted = getKnowledgeElementByName(changedFile.getOldName());
+		deleteKnowledgeElement(fileToBeDeleted, null);
+	}
+
+	private void handleRename(ChangedFile changedFile) {
+		handleDelete(changedFile);
+		handleAdd(changedFile);
+	}
+
 	public void extractAllChangedFiles() {
 		GitClient gitClient = GitClient.getOrCreate(projectKey);
 		Diff diff = gitClient.getDiffOfEntireDefaultBranch();

src/main/java/de/uhd/ifi/se/decision/management/jira/rest/ViewRest.java

@@ -37,7 +37,6 @@
 import de.uhd.ifi.se.decision.management.jira.model.KnowledgeType;
 import de.uhd.ifi.se.decision.management.jira.persistence.ConfigPersistenceManager;
 import de.uhd.ifi.se.decision.management.jira.persistence.KnowledgePersistenceManager;
-import de.uhd.ifi.se.decision.management.jira.persistence.singlelocations.CodeClassPersistenceManager;
 import de.uhd.ifi.se.decision.management.jira.view.decisionguidance.Recommendation;
 import de.uhd.ifi.se.decision.management.jira.view.decisionguidance.RecommendationEvaluation;
 import de.uhd.ifi.se.decision.management.jira.view.decisiontable.DecisionTable;
@@ -88,7 +87,6 @@ public Response getElementsOfFeatureBranchForJiraIssue(@Context HttpServletReque
 					.entity(ImmutableMap.of("error", "Git extraction is disabled in project settings.")).build();
 		}
 		new CommitMessageToCommentTranscriber(issue).postCommitsIntoJiraIssueComments();
-		new CodeClassPersistenceManager(projectKey).extractAllChangedFiles();
 		return Response.ok(new DiffViewer(projectKey, issueKey)).build();
 	}
 

src/test/java/de/uhd/ifi/se/decision/management/jira/extraction/gitclient/TestGetBranches.java

@@ -6,17 +6,10 @@
 
 import org.eclipse.jgit.api.Git;
 import org.eclipse.jgit.lib.Ref;
-import org.junit.Before;
 import org.junit.Test;
 
 public class TestGetBranches extends TestSetUpGit {
 
-	@Override
-	@Before
-	public void setUp() {
-		super.setUp();
-	}
-
 	@Test
 	public void testGetAllBranches() {
 		List<Ref> remoteBranches = gitClient.getBranches();

src/test/java/de/uhd/ifi/se/decision/management/jira/extraction/gitclient/TestSetUpGit.java

@@ -226,7 +226,7 @@ private static void makeExampleCommit(File inputFile, String targetName, String
 		}
 	}
 
-	private static void makeExampleCommit(String filename, String content, String commitMessage) {
+	public static void makeExampleCommit(String filename, String content, String commitMessage) {
 		Git git = gitClient.getGitClientsForSingleRepo(GIT_URI).getGit();
 		try {
 			File inputFile = new File(gitClient.getGitClientsForSingleRepo(GIT_URI).getGitDirectory().getParent(),

src/test/java/de/uhd/ifi/se/decision/management/jira/persistence/codeclasspersistencemanager/TestGetKnowledgeElement.java

@@ -80,10 +80,4 @@ public void testGetEntryForKnowledgeElement() {
 	public void testGetKnowledgeElements() {
 		assertEquals(1, codeClassPersistenceManager.getKnowledgeElements().size());
 	}
-
-	@Test
-	@NonTransactional
-	public void testGetKnowledgeElementsMatchingName() {
-		assertEquals(1, codeClassPersistenceManager.getKnowledgeElementsMatchingName(classElement.getSummary()).size());
-	}
 }

src/test/java/de/uhd/ifi/se/decision/management/jira/persistence/codeclasspersistencemanager/TestMaintainChangedFilesInDatabase.java

@@ -20,7 +20,7 @@ public class TestMaintainChangedFilesInDatabase extends TestSetUpGit {
 	@Override
 	@Before
 	public void setUp() {
-		super.setUp();
+		init();
 		codeClassPersistenceManager = new CodeClassPersistenceManager("TEST");
 		diff = gitClient.getDiffOfEntireDefaultBranch();
 	}