Skip to content

Commit c2cd1cf

Browse files
JorenSixJorenSix
committed
Caching improvements: to avoid repeated feature extraction caching is now implemented with a content based hash and not on a path of a file.
1 parent 70a3168 commit c2cd1cf

22 files changed

+1021
-372
lines changed

‎resources/defaults/logging.properties

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,4 +55,5 @@ java.util.logging.FileHandler.limit=5000000
5555
# integer to the base file name:
5656
java.util.logging.FileHandler.count=5
5757

58-
# Style of output (Simple or XML):
58+
# Style of output (Simple or XML):
59+
java.util.logging.FileHandler.formatter = java.util.logging.SimpleFormatter

‎src/main/java/be/panako/cli/Application.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -140,9 +140,9 @@ protected boolean checkFile(String file){
140140
File f = new File(file);
141141
boolean fileOk = false;
142142
if(f.exists() && f.canRead()){
143-
fileOk = true;
143+
fileOk = FileUtils.checkFileSize(f,Config.getInt(Key.MAX_FILE_SIZE));
144144
}else{
145-
String message = "Could not read " + f.getAbsolutePath() + " it does not exist or is not accesible at the moment.)";
145+
String message = "Could not read " + f.getAbsolutePath() + " it does not exist or is not accessible at the moment.)";
146146
LOG.warning(message);
147147
System.out.println(message);
148148
}

‎src/main/java/be/panako/cli/Delete.java

Lines changed: 21 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -45,10 +45,7 @@
4545

4646
import be.panako.strategy.Strategy;
4747
import be.panako.strategy.olaf.OlafStrategy;
48-
import be.panako.util.Config;
49-
import be.panako.util.Key;
50-
import be.panako.util.StopWatch;
51-
import be.panako.util.TimeUnit;
48+
import be.panako.util.*;
5249

5350
/**
5451
* Delete fingerptings from the index.
@@ -121,42 +118,29 @@ public DeleteTask(File file,int taskID,int totalTasks){
121118
public void run() {
122119

123120
StopWatch w = new StopWatch();
124-
if(checkFile(file)){
125-
126-
Strategy strategy = Strategy.getInstance();
127-
128-
boolean hasResource = false;
129-
hasResource = strategy.hasResource(file.getAbsolutePath());
130-
131-
String message=null;
132-
if(hasResource){
133-
message = String.format("%d/%d;%s;%s;%s",taskID,totalTasks,file.getName(),StopWatch.toTime("", 0),"Deletion skipped: resource not in the key value store;");
134-
}else{
135-
double durationInSeconds = strategy.delete(file.getAbsolutePath());
136-
137-
double cpuSecondsPassed = w.timePassed(TimeUnit.SECONDS);
138-
String audioDuration = StopWatch.toTime("", (int) Math.round(durationInSeconds));
139-
String cpuTimeDuration = w.formattedToString();
140-
double timeRatio = durationInSeconds/cpuSecondsPassed;
141-
message = String.format("%d/%d;%s;%s;%s;%.2f",taskID,totalTasks,file.getName(),audioDuration,cpuTimeDuration,timeRatio);
142-
}
143-
LOG.info(message);
144-
System.out.println(message);
145-
}
146-
}
147-
148-
private boolean checkFile(File file){
149-
boolean fileOk = false;
150-
//file must be smaller than a configured number of bytes
151-
if(file.length() != 0 && file.length() < Config.getInt(Key.MAX_FILE_SIZE)){
152-
fileOk = true;
121+
Strategy strategy = Strategy.getInstance();
122+
123+
boolean hasResource = false;
124+
hasResource = strategy.hasResource(file.getAbsolutePath());
125+
126+
String message=null;
127+
if(hasResource){
128+
message = String.format("%d/%d;%s;%s;%s",taskID,totalTasks,file.getName(),StopWatch.toTime("", 0),"Deletion skipped: resource not in the key value store;");
153129
}else{
154-
String message = "Could not process " + file.getName() + " it has an unacceptable file size: zero or larger than " + Config.getInt(Key.MAX_FILE_SIZE) + "bytes ).";
155-
LOG.warning(message);
156-
System.out.println(message);
130+
double durationInSeconds = strategy.delete(file.getAbsolutePath());
131+
132+
double cpuSecondsPassed = w.timePassed(TimeUnit.SECONDS);
133+
String audioDuration = StopWatch.toTime("", (int) Math.round(durationInSeconds));
134+
String cpuTimeDuration = w.formattedToString();
135+
double timeRatio = durationInSeconds/cpuSecondsPassed;
136+
message = String.format("%d/%d;%s;%s;%s;%.2f",taskID,totalTasks,file.getName(),audioDuration,cpuTimeDuration,timeRatio);
157137
}
158-
return fileOk;
138+
LOG.info(message);
139+
System.out.println(message);
140+
159141
}
142+
143+
160144
}
161145

162146

‎src/main/java/be/panako/cli/Monitor.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -83,14 +83,14 @@ public void run(String... args) {
8383
if(hasArgument("debug", args) || processors==1){
8484
int taskNumber = 1;
8585
for(File file: files){
86-
new Monitor.MonitorTask(file.getPath(),taskNumber,files.size()).run();
86+
new Monitor.MonitorTask(file.getAbsolutePath(),taskNumber,files.size()).run();
8787
taskNumber++;
8888
}
8989
}else{
9090
ExecutorService executor = Executors.newFixedThreadPool(processors);
9191
int taskNumber = 1;
9292
for(File file: files){
93-
executor.submit(new Monitor.MonitorTask(file.getPath(),taskNumber,files.size()));
93+
executor.submit(new Monitor.MonitorTask(file.getAbsolutePath(),taskNumber,files.size()));
9494
taskNumber++;
9595
}
9696
executor.shutdown();

‎src/main/java/be/panako/cli/Print.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ public void run(String... args) {
5959
Strategy strategy = Strategy.getInstance();
6060

6161
for(File file: files){
62-
strategy.print(file.getPath(),sonicVisualizerOutput);
62+
strategy.print(file.getAbsolutePath(),sonicVisualizerOutput);
6363
}
6464

6565
}

‎src/main/java/be/panako/cli/Query.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -70,14 +70,14 @@ public void run(String... args) {
7070
if(hasArgument("debug", args) || processors==1){
7171
int taskNumber = 1;
7272
for(File file: files){
73-
new QueryTask(file.getPath(),taskNumber,files.size()).run();
73+
new QueryTask(file.getAbsolutePath(),taskNumber,files.size()).run();
7474
taskNumber++;
7575
}
7676
}else{
7777
ExecutorService executor = Executors.newFixedThreadPool(processors);
7878
int taskNumber = 1;
7979
for(File file: files){
80-
executor.submit(new QueryTask(file.getPath(),taskNumber,files.size()));
80+
executor.submit(new QueryTask(file.getAbsolutePath(),taskNumber,files.size()));
8181
taskNumber++;
8282
}
8383
executor.shutdown();

‎src/main/java/be/panako/cli/Resolve.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,11 +47,11 @@ class Resolve extends Application {
4747

4848
@Override
4949
public void run(String... args) {
50-
Strategy strat = Strategy.getInstance();
50+
Strategy strategy = Strategy.getInstance();
5151
List<File> files = getFilesFromArguments(args);
5252

5353
for(File f : files) {
54-
System.out.println(strat.resolve(f.getPath()));
54+
System.out.println(strategy.resolve(f.getAbsolutePath()));
5555
}
5656
}
5757

‎src/main/java/be/panako/cli/Store.java

Lines changed: 20 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -42,10 +42,7 @@
4242
import java.util.logging.Logger;
4343

4444
import be.panako.strategy.Strategy;
45-
import be.panako.util.Config;
46-
import be.panako.util.Key;
47-
import be.panako.util.StopWatch;
48-
import be.panako.util.TimeUnit;
45+
import be.panako.util.*;
4946

5047
/**
5148
* Store audio fingerptings in the storage.
@@ -116,49 +113,31 @@ public StoreTask(File file,int taskID,int totalTasks){
116113

117114
@Override
118115
public void run() {
119-
120116
StopWatch w = new StopWatch();
121-
if(checkFile(file)){
122-
123-
Strategy strategy = Strategy.getInstance();
124-
125-
boolean isDouble = false;
126-
if(Config.getBoolean(Key.CHECK_DUPLICATE_FILE_NAMES) ){
127-
isDouble = strategy.hasResource(file.getAbsolutePath());
128-
}
117+
Strategy strategy = Strategy.getInstance();
129118

130-
String message=null;
131-
if(isDouble){
132-
message = String.format("%d/%d;%s;%s",taskID,totalTasks,file.getName(),"Skipped: resource already stored;");
133-
}else{
134-
double durationInSeconds = strategy.store(file.getAbsolutePath(), file.getName());
135-
double cpuSecondsPassed = w.timePassed(TimeUnit.SECONDS);
136-
String audioDuration = StopWatch.toTime("", (int) Math.round(durationInSeconds));
137-
String cpuTimeDuration = w.formattedToString();
138-
double timeRatio = durationInSeconds/cpuSecondsPassed;
139-
message = String.format("%d/%d;%s;%s;%s;%.2f",taskID,totalTasks,file.getName(),audioDuration,cpuTimeDuration,timeRatio);
140-
}
141-
LOG.info(message);
142-
System.out.println(message);
119+
boolean isDouble = false;
120+
if(Config.getBoolean(Key.CHECK_DUPLICATE_FILE_NAMES) ){
121+
isDouble = strategy.hasResource(file.getAbsolutePath());
143122
}
144-
}
145-
146-
private boolean checkFile(File file){
147-
boolean fileOk = false;
148-
149-
//file must be smaller than a configured number of bytes
150-
long maxFileSize = Config.getInt(Key.MAX_FILE_SIZE);
151-
//from megabytes to bytes
152-
maxFileSize = maxFileSize * 1024 * 1024;
153-
if(file.length() != 0 && file.length() < maxFileSize ){
154-
fileOk = true;
123+
124+
String message=null;
125+
if(isDouble){
126+
message = String.format("%d/%d;%s;%s",taskID,totalTasks,file.getName(),"Skipped: resource already stored;");
155127
}else{
156-
String message = "Could not process " + file.getName() + " it has an unacceptable file size.\n\tFile is " + file.length() + " bytes. \n\tShould be more than zero and smaller than " + Config.getInt(Key.MAX_FILE_SIZE) + " bytes ).";
157-
LOG.warning(message);
158-
System.out.println(message);
128+
double durationInSeconds = strategy.store(file.getAbsolutePath(), file.getName());
129+
double cpuSecondsPassed = w.timePassed(TimeUnit.SECONDS);
130+
String audioDuration = StopWatch.toTime("", (int) Math.round(durationInSeconds));
131+
String cpuTimeDuration = w.formattedToString();
132+
double timeRatio = durationInSeconds/cpuSecondsPassed;
133+
message = String.format("%d/%d;%s;%s;%s;%.2f",taskID,totalTasks,file.getName(),audioDuration,cpuTimeDuration,timeRatio);
159134
}
160-
return fileOk;
135+
LOG.info(message);
136+
System.out.println(message);
137+
161138
}
139+
140+
162141
}
163142

164143

0 commit comments

Comments
 (0)