Skip to content

Commit 13b4e79

Browse files
pauloborbapauloborba
authored
Merge pull request #199 from jpedroh/fix-unstructured-merge
Adjustments in Last Merge experiment
2 parents c453657 + 945b315 commit 13b4e79

47 files changed

Lines changed: 1118 additions & 976 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎dependencies/spork.jar‎

27.5 MB
Binary file not shown.

‎src/main/app/MiningWorker.groovy‎

Lines changed: 22 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,37 @@
11
package app
22

3+
import exception.UnstagedChangesException
4+
import interfaces.CommitFilter
5+
import interfaces.DataCollector
6+
import org.apache.logging.log4j.LogManager
7+
import org.apache.logging.log4j.Logger
8+
import project.MergeCommit
9+
import project.Project
10+
import project.commitHashesExtraction.CommitHashesExtractor
11+
import services.util.Utils
12+
import util.FileManager
13+
import util.MergeHelper
14+
import util.ProcessRunner
315
import java.util.Collections
416
import java.util.Random
517
import java.text.SimpleDateFormat
618

7-
import static app.MiningFramework.arguments
19+
import java.text.SimpleDateFormat
820
import java.util.concurrent.BlockingQueue
921

10-
import project.*
11-
import interfaces.*
12-
import exception.UnstagedChangesException
13-
import util.*
14-
15-
import services.util.Utils;
22+
import static app.MiningFramework.arguments
1623

1724
class MiningWorker implements Runnable {
25+
private static Logger LOG = LogManager.getLogger(MiningWorker.class)
1826

1927
private Set<DataCollector> dataCollectors
2028
private CommitFilter commitFilter
2129
private BlockingQueue<Project> projectList
2230
private String baseDir
31+
private CommitHashesExtractor commitHashesExtractor
2332

2433
MiningWorker(Set<DataCollector> dataCollectors, CommitFilter commitFilter, BlockingQueue<Project> projectList, String baseDir) {
34+
this.commitHashesExtractor = CommitHashesExtractor.Factory.build()
2535
this.dataCollectors = dataCollectors
2636
this.commitFilter = commitFilter
2737
this.projectList = projectList
@@ -41,7 +51,7 @@ class MiningWorker implements Runnable {
4151
checkForUnstagedChanges(project);
4252
}
4353

44-
def (mergeCommits, skipped) = project.getMergeCommits(arguments.getSinceDate(), arguments.getUntilDate())
54+
def (mergeCommits, skipped) = project.getMergeCommits(commitHashesExtractor)
4555

4656
Random random = new Random(arguments.getRandomSeed())
4757
Collections.shuffle(mergeCommits, random)
@@ -97,7 +107,7 @@ class MiningWorker implements Runnable {
97107
}
98108

99109
private void cloneRepository(Project project, String target) {
100-
println "Cloning repository ${project.getName()} into ${target}"
110+
LOG.info("Cloning repository ${project.getName()} into ${target}")
101111

102112
File projectDirectory = new File(target)
103113
if (projectDirectory.exists()) {
@@ -114,10 +124,11 @@ class MiningWorker implements Runnable {
114124
}
115125

116126
ProcessBuilder builder = ProcessRunner.buildProcess('./', 'git', 'clone', url, target)
117-
builder.redirectOutput(ProcessBuilder.Redirect.INHERIT)
118-
119127
Process process = ProcessRunner.startProcess(builder)
128+
process.getInputStream().eachLine(LOG::trace)
129+
process.getErrorStream().eachLine(LOG::warn)
120130
process.waitFor()
131+
LOG.info("Finished cloning repository ${project.getName()} into ${target}")
121132

122133
project.setPath(target)
123134
}

‎src/main/arguments/ArgsParser.groovy‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ class ArgsParser {
3939
this.cli.e(longOpt: 'extension', args: 1, argName: 'file extenson', 'Specify the file extension that should be used in the analysis (e.g. .rb, .ts, .java, .cpp. Default: .java)')
4040
this.cli.l(longOpt: 'language-separators', args: 1, argName: 'language syntactic separators', 'Specify the language separators that should be used in the analysis. Required for (and only considered when) running studies with the CSDiff tool. Default: \"{ } ( ) ; ,\"')
4141
this.cli.log(longOpt: 'log-level', args: 1, argName: 'log level', 'Specify the minimum log level: (OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE, ALL). Default: \"INFO\"')
42+
this.cli.pchf(longOpt: 'project-commit-hashes-file', args: 1, argName: 'project commit hashes file', 'Path to a file with projects and commits to be analyzed')
4243
}
4344

4445
Arguments parse(args) {
@@ -142,6 +143,10 @@ class ArgsParser {
142143
if(this.options.log) {
143144
args.setLogLevel(Level.toLevel(this.options.log))
144145
}
146+
147+
if (this.options.pchf) {
148+
args.setProjectCommitHashesFile(this.options.pchf)
149+
}
145150
}
146151

147152
private boolean repositoryExists(String repositoryURL) {

‎src/main/arguments/Arguments.groovy‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ class Arguments {
2121
private String syntacticSeparators
2222
private String fileExtension
2323
private Level logLevel
24+
private String projectCommitHashesFile
2425

2526
Arguments() { // set the default values for all parameters
2627
randomSeed = 1
@@ -37,6 +38,7 @@ class Arguments {
3738
syntacticSeparators = '{ } ( ) ; ,'
3839
fileExtension = '.java'
3940
logLevel = Level.INFO
41+
projectCommitHashesFile = ''
4042
}
4143

4244
void setRandomSeed(int randomSeed) {
@@ -167,4 +169,12 @@ class Arguments {
167169
this.logLevel = logLevel
168170
Configurator.setRootLevel(logLevel)
169171
}
172+
173+
String getProjectCommitHashesFile() {
174+
return projectCommitHashesFile
175+
}
176+
177+
void setProjectCommitHashesFile(String projectCommitHashesFile) {
178+
this.projectCommitHashesFile = projectCommitHashesFile
179+
}
170180
}

‎src/main/injectors/GenericMergeModule.groovy‎

Lines changed: 58 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -6,51 +6,77 @@ import interfaces.CommitFilter
66
import interfaces.DataCollector
77
import interfaces.OutputProcessor
88
import interfaces.ProjectProcessor
9-
import org.apache.logging.log4j.LogManager
10-
import org.apache.logging.log4j.Logger
119
import services.commitFilters.MutuallyModifiedFilesCommitFilter
12-
import services.dataCollectors.GenericMerge.GenericMergeConfig
13-
import services.dataCollectors.GenericMerge.GenericMergeDataCollector
14-
import services.dataCollectors.GenericMerge.MergeConflictsComparator
15-
import services.dataCollectors.GenericMerge.MergeToolsComparator
16-
import services.dataCollectors.GenericMerge.ScenarioLOCsCounter
17-
import services.dataCollectors.GenericMerge.UnstructuredMergeCollector
18-
import services.outputProcessors.GenericMergeDataOutputProcessor
10+
import services.dataCollectors.buildRequester.RequestBuildForRevisionWithFilesDataCollector
11+
import services.dataCollectors.common.CompareScenarioMergeConflictsDataCollector
12+
import services.dataCollectors.common.RunDataCollectorsInParallel
13+
import services.dataCollectors.common.SyntacticallyCompareScenarioFilesDataCollector
14+
import services.dataCollectors.fileSyntacticNormalization.FormatFileSyntacticNormalizationDataCollector
15+
import services.dataCollectors.fileSyntacticNormalization.JDimeFileSyntacticNormalizationDataCollector
16+
import services.dataCollectors.fileSyntacticNormalization.SporkFileSyntacticNormalizationDataCollector
17+
import services.dataCollectors.mergeToolExecutors.JDimeMergeToolExecutorDataCollector
18+
import services.dataCollectors.mergeToolExecutors.LastMergeMergeToolExecutorDataCollector
19+
import services.dataCollectors.mergeToolExecutors.MergirafMergeToolExecutorDataCollector
20+
import services.dataCollectors.mergeToolExecutors.SporkMergeToolExecutorDataCollector
21+
import services.outputProcessors.EmptyOutputProcessor
1922
import services.projectProcessors.DummyProjectProcessor
20-
import services.util.ci.CIPlatform
21-
import services.util.ci.TravisPlatform
22-
23-
import java.nio.file.Files
24-
import java.nio.file.Paths
2523

2624
class GenericMergeModule extends AbstractModule {
27-
private static Logger LOG = LogManager.getLogger(GenericMergeModule.class)
28-
2925
@Override
3026
protected void configure() {
3127
Multibinder<ProjectProcessor> projectProcessorBinder = Multibinder.newSetBinder(binder(), ProjectProcessor.class)
3228
projectProcessorBinder.addBinding().to(DummyProjectProcessor.class)
3329

3430
Multibinder<DataCollector> dataCollectorBinder = Multibinder.newSetBinder(binder(), DataCollector.class)
35-
dataCollectorBinder.addBinding().to(ScenarioLOCsCounter.class)
36-
dataCollectorBinder.addBinding().to(GenericMergeDataCollector.class)
37-
dataCollectorBinder.addBinding().to(MergeToolsComparator.class)
38-
dataCollectorBinder.addBinding().to(MergeConflictsComparator.class)
39-
dataCollectorBinder.addBinding().to(UnstructuredMergeCollector.class)
4031

41-
Multibinder<OutputProcessor> outputProcessorBinder = Multibinder.newSetBinder(binder(), OutputProcessor.class)
42-
outputProcessorBinder.addBinding().to(GenericMergeDataOutputProcessor.class)
32+
// Run the merge tools on the scenarios
33+
dataCollectorBinder.addBinding().to(JDimeMergeToolExecutorDataCollector.class)
34+
dataCollectorBinder.addBinding().to(LastMergeMergeToolExecutorDataCollector.class)
35+
dataCollectorBinder.addBinding().to(SporkMergeToolExecutorDataCollector.class)
36+
dataCollectorBinder.addBinding().to(MergirafMergeToolExecutorDataCollector.class)
4337

44-
bind(CommitFilter.class).to(MutuallyModifiedFilesCommitFilter.class)
45-
bind(CIPlatform.class).to(TravisPlatform.class)
38+
// Run, in parallel, normalizations on resulting files
39+
dataCollectorBinder.addBinding().toInstance(new RunDataCollectorsInParallel([
40+
new JDimeFileSyntacticNormalizationDataCollector("merge.java", "merge.jdime_normalized.java"),
41+
new JDimeFileSyntacticNormalizationDataCollector("merge.last_merge.java", "merge.last_merge.jdime_normalized.java"),
42+
new SporkFileSyntacticNormalizationDataCollector("merge.java", "merge.spork_normalized.java"),
43+
new SporkFileSyntacticNormalizationDataCollector("merge.spork.java", "merge.spork.spork_normalized.java"),
44+
new SporkFileSyntacticNormalizationDataCollector("merge.mergiraf.java", "merge.mergiraf.spork_normalized.java"),
45+
]))
4646

47-
createExecutionReportsFile()
48-
}
47+
// Run, in parallel, syntactical comparisons between files and textual comparison between commits
48+
dataCollectorBinder.addBinding().toInstance(new RunDataCollectorsInParallel([
49+
// Syntactic comparison with merge commits
50+
new SyntacticallyCompareScenarioFilesDataCollector("merge.java", "merge.last_merge.java"),
51+
new SyntacticallyCompareScenarioFilesDataCollector("merge.jdime_normalized.java", "merge.jdime.java"),
52+
new SyntacticallyCompareScenarioFilesDataCollector("merge.spork_normalized.java", "merge.spork.spork_normalized.java"),
53+
new SyntacticallyCompareScenarioFilesDataCollector("merge.java", "merge.mergiraf.java"),
54+
55+
// Syntactic comparison between tools themselves
56+
new SyntacticallyCompareScenarioFilesDataCollector("merge.jdime.java", "merge.last_merge.jdime_normalized.java"),
57+
new SyntacticallyCompareScenarioFilesDataCollector("merge.mergiraf.spork_normalized.java", "merge.spork.spork_normalized.java"),
58+
new SyntacticallyCompareScenarioFilesDataCollector("merge.mergiraf.java", "merge.last_merge.java"),
59+
60+
// Conflicts comparison between tools themselves
61+
new CompareScenarioMergeConflictsDataCollector("merge.jdime.java", "merge.last_merge.java"),
62+
new CompareScenarioMergeConflictsDataCollector("merge.mergiraf.java", "merge.spork.java"),
63+
new CompareScenarioMergeConflictsDataCollector("merge.mergiraf.java", "merge.last_merge.java"),
64+
]))
4965

50-
private static void createExecutionReportsFile() {
51-
LOG.info("Creating Generic Merge report file")
52-
Files.createDirectories(Paths.get(GenericMergeConfig.GENERIC_MERGE_REPORT_PATH))
53-
def reportFile = new File(GenericMergeConfig.GENERIC_MERGE_REPORT_FILE_NAME)
54-
reportFile.createNewFile()
66+
dataCollectorBinder.addBinding().toInstance(new RunDataCollectorsInParallel([new FormatFileSyntacticNormalizationDataCollector("merge.last_merge.java", "merge.last_merge.format_normalized.java"),
67+
new FormatFileSyntacticNormalizationDataCollector("merge.mergiraf.java", "merge.mergiraf.format_normalized.java")]))
68+
69+
dataCollectorBinder.addBinding().toInstance(new SyntacticallyCompareScenarioFilesDataCollector("merge.mergiraf.format_normalized.java", "merge.last_merge.format_normalized.java"))
70+
71+
72+
dataCollectorBinder.addBinding().toInstance(new RequestBuildForRevisionWithFilesDataCollector("merge.last_merge.java"))
73+
dataCollectorBinder.addBinding().toInstance(new RequestBuildForRevisionWithFilesDataCollector("merge.jdime.java"))
74+
dataCollectorBinder.addBinding().toInstance(new RequestBuildForRevisionWithFilesDataCollector("merge.mergiraf.java"))
75+
dataCollectorBinder.addBinding().toInstance(new RequestBuildForRevisionWithFilesDataCollector("merge.spork.java"))
76+
77+
Multibinder<OutputProcessor> outputProcessorBinder = Multibinder.newSetBinder(binder(), OutputProcessor.class)
78+
outputProcessorBinder.addBinding().to(EmptyOutputProcessor.class)
79+
80+
bind(CommitFilter.class).to(MutuallyModifiedFilesCommitFilter.class)
5581
}
5682
}

‎src/main/project/Project.groovy‎

Lines changed: 30 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,15 @@
11
package project
22

3-
import util.ProcessRunner
43
import exception.UnexpectedOutputException
4+
import org.apache.logging.log4j.LogManager
5+
import project.commitHashesExtraction.CommitHashesExtractor
6+
import util.ProcessRunner
57

6-
import java.util.Collections
7-
import java.util.Random
8-
import java.util.regex.Pattern
98
import java.util.regex.Matcher
9+
import java.util.regex.Pattern
1010

1111
class Project {
12+
private static LOG = LogManager.getLogger(Project.class)
1213

1314
private String name
1415
private String path
@@ -42,49 +43,41 @@ class Project {
4243
return matcher.find()
4344
}
4445

45-
List getMergeCommits(String sinceDate, String untilDate) {
46+
List getMergeCommits(CommitHashesExtractor commitHashesExtractor) {
4647
ArrayList<String> skipped = new ArrayList<String>()
4748
ArrayList<MergeCommit> mergeCommits = new ArrayList<MergeCommit>()
48-
49-
Process gitLog = constructAndRunGitLog(sinceDate, untilDate)
50-
def expectedOutput = ~/.*-(.* .*)+/
51-
gitLog.getInputStream().eachLine {
5249

53-
// Each line contains the hash of the commit followed by the hashes of the parents.
54-
if(it ==~ expectedOutput) {
55-
56-
String[] informations = it.split('-') // <commit hash>-<parents hash>
57-
String SHA = getSHA(informations)
58-
String[] parentsSHA = getParentsSHA(informations)
50+
commitHashesExtractor.extractCommitHashes(this).each(commitHashes -> {
51+
def SHA = commitHashes.mergeSha
52+
def parentsSHA = commitHashes.parents
5953

54+
try {
55+
String ancestorSHA = getCommonAncestor(SHA, parentsSHA)
56+
MergeCommit mergeCommit = new MergeCommit(SHA, parentsSHA, ancestorSHA)
57+
mergeCommits.add(mergeCommit)
58+
} catch (UnexpectedOutputException e) {
59+
println e.message
6060
try {
61+
println "Trying to fetch it"
62+
// Let's try fetching it before skipping it
63+
constructAndRunGitFetch(SHA)
6164
String ancestorSHA = getCommonAncestor(SHA, parentsSHA)
6265
MergeCommit mergeCommit = new MergeCommit(SHA, parentsSHA, ancestorSHA)
6366
mergeCommits.add(mergeCommit)
64-
} catch (UnexpectedOutputException e) {
67+
} catch (UnexpectedOutputException ex) {
6568
println "Skipping merge commit ${SHA}"
66-
println e.message
69+
println ex.message
6770
skipped.add(SHA)
6871
}
69-
} else {
70-
throw new UnexpectedOutputException('Git log returned an unexpected output. Could not retrieve merge commits.', '<commit hash>-<parents hash>', it)
7172
}
72-
}
73-
73+
})
74+
7475
if(mergeCommits.isEmpty())
7576
println "No merge commits."
7677

7778
return [mergeCommits, skipped]
7879
}
7980

80-
private String getSHA(String[] informations) {
81-
return informations[0]
82-
}
83-
84-
private String[] getParentsSHA(String[] informations) {
85-
return informations[1].split(' ')
86-
}
87-
8881
private String getCommonAncestor(mergeCommitSHA, parentsSHA) {
8982
Process gitMergeBase = constructAndRunGitMergeBase(mergeCommitSHA, parentsSHA)
9083
def expectedOutput = ~/[0-9a-z]{7,}/
@@ -96,6 +89,14 @@ class Project {
9689
throw new UnexpectedOutputException('Git merge-base returned an unexpected output. Could not retrieve the ancestor commit.', '<commit hash>', actualOutput)
9790
}
9891

92+
private void constructAndRunGitFetch(String mergeCommitSHA) {
93+
ProcessBuilder gitMergeBaseBuilder = ProcessRunner.buildProcess(path, 'git', 'fetch', 'origin', mergeCommitSHA)
94+
def exitCode = ProcessRunner.startProcess(gitMergeBaseBuilder).waitFor()
95+
if (exitCode > 0) {
96+
throw new UnexpectedOutputException("Could not fetch merge commit ${mergeCommitSHA} from origin", "Exit code 0", "Exit code ${exitCode}")
97+
}
98+
}
99+
99100
private Process constructAndRunGitMergeBase(String mergeCommitSHA, String[] parentsSHA) {
100101
ProcessBuilder gitMergeBaseBuilder = ProcessRunner.buildProcess(path, 'git', 'merge-base')
101102
if (parentsSHA.length > 2)
@@ -105,15 +106,6 @@ class Project {
105106
return ProcessRunner.startProcess(gitMergeBaseBuilder)
106107
}
107108

108-
private Process constructAndRunGitLog(String sinceDate, String untilDate) {
109-
ProcessBuilder gitLogBuilder = ProcessRunner.buildProcess(path, 'git', '--no-pager', 'log', '--merges', '--pretty=%H-%p')
110-
if(!sinceDate.equals(''))
111-
ProcessRunner.addCommand(gitLogBuilder, "--since=\"${sinceDate}\"")
112-
if(!untilDate.equals(''))
113-
ProcessRunner.addCommand(gitLogBuilder, "--until=\"${untilDate}\"")
114-
return ProcessRunner.startProcess(gitLogBuilder)
115-
}
116-
117109
String getName() {
118110
return name
119111
}

‎src/main/project/commitHashesExtraction/CommitHashesExtractor.groovy‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
package project.commitHashesExtraction
2+
3+
import project.Project
4+
5+
import static app.MiningFramework.arguments
6+
7+
interface CommitHashesExtractor {
8+
class CommitHashes {
9+
public String mergeSha
10+
public String[] parents
11+
12+
CommitHashes(String mergeSha, String... parents) {
13+
this.mergeSha = mergeSha
14+
this.parents = parents
15+
}
16+
}
17+
18+
class Factory {
19+
static CommitHashesExtractor build() {
20+
if (arguments.projectCommitHashesFile != '') {
21+
def file = new File(arguments.projectCommitHashesFile)
22+
if (!file.exists()) {
23+
throw new RuntimeException("File ${file.path} not found")
24+
}
25+
return new ProjectCommitListCommitHashesExtractor(file)
26+
}
27+
28+
return new GitLogCommitHashesExtractor(arguments.sinceDate, arguments.untilDate)
29+
}
30+
}
31+
32+
List<CommitHashes> extractCommitHashes(Project project)
33+
}

0 commit comments

Comments
 (0)