Skip to content

Commit 60b0023

Browse files
BerndDoserBerndDoser
committed
multiple changes
- update mapping binary file format to version 2 - update best rotation and flip binary file format to version 2 - initialize SOM
1 parent 393f02a commit 60b0023

6 files changed

Lines changed: 138 additions & 43 deletions

File tree

‎scripts/rotations.py‎

Lines changed: 18 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -54,11 +54,21 @@ def readRotations(self):
5454
#Unpacks the map parameters
5555
inputStream = open(self.__fileName, 'rb')
5656
tools.ignore_header_comments(inputStream)
57-
58-
self.__numberOfImages = struct.unpack("i", inputStream.read(4))[0]
59-
self.__somWidth = struct.unpack("i", inputStream.read(4))[0]
60-
self.__somHeight = struct.unpack("i", inputStream.read(4))[0]
61-
self.__somDepth = struct.unpack("i", inputStream.read(4))[0]
57+
58+
# <file format version> 3 <number of entries> <som layout> <data>
59+
version, file_type, number_of_data_entries, som_layout, som_dimensionality = struct.unpack('i' * 5, inputStream.read(4 * 5))
60+
print('version:', version)
61+
print('file_type:', file_type)
62+
print('number_of_data_entries:', number_of_data_entries)
63+
print('som_layout:', som_layout)
64+
print('som dimensionality:', som_dimensionality)
65+
som_dimensions = struct.unpack('i' * som_dimensionality, inputStream.read(4 * som_dimensionality))
66+
print('som dimensions:', som_dimensions)
67+
68+
self.__numberOfImages = number_of_data_entries
69+
self.__somWidth = som_dimensions[0]
70+
self.__somHeight = som_dimensions[1] if som_dimensionality > 1 else 1
71+
self.__somDepth = som_dimensions[2] if som_dimensionality > 2 else 1
6272

6373
print ("images: " + str(self.__numberOfImages))
6474
print ("width: " + str(self.__somWidth))
@@ -107,11 +117,11 @@ def print_usage():
107117
print_usage()
108118
sys.exit(1)
109119

110-
#Default parameters
111-
image = -1
120+
# Default parameters
121+
image = 0
112122
mapping = ""
113123

114-
#Use inputted parameters
124+
# Use input parameters
115125
for opt, arg in opts:
116126
if opt in ("-i", "--image"):
117127
image = int(arg)

‎scripts/show_heatmap.py‎

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -37,12 +37,23 @@ def getHeatmap(self, objectNumber):
3737
def readMap(self):
3838
#Unpacks the map parameters
3939
inputStream = open(self.__fileName, 'rb')
40-
tools.ignore_header_comments(inputStream) # find end of header
41-
42-
self.__numberOfImages = struct.unpack("i", inputStream.read(4))[0]
43-
self.__somWidth = struct.unpack("i", inputStream.read(4))[0]
44-
self.__somHeight = struct.unpack("i", inputStream.read(4))[0]
45-
self.__somDepth = struct.unpack("i", inputStream.read(4))[0]
40+
tools.ignore_header_comments(inputStream)
41+
42+
# <file format version> 2 <data-type> <number of entries> <som layout> <data>
43+
version, file_type, data_type, number_of_data_entries, som_layout, som_dimensionality = struct.unpack('i' * 6, inputStream.read(4 * 6))
44+
print('version:', version)
45+
print('file_type:', file_type)
46+
print('data_type:', data_type)
47+
print('number_of_data_entries:', number_of_data_entries)
48+
print('som_layout:', som_layout)
49+
print('som dimensionality:', som_dimensionality)
50+
som_dimensions = struct.unpack('i' * som_dimensionality, inputStream.read(4 * som_dimensionality))
51+
print('som dimensions:', som_dimensions)
52+
53+
self.__numberOfImages = number_of_data_entries
54+
self.__somWidth = som_dimensions[0]
55+
self.__somHeight = som_dimensions[1] if som_dimensionality > 1 else 1
56+
self.__somDepth = som_dimensions[2] if som_dimensionality > 2 else 1
4657

4758
print ("images: " + str(self.__numberOfImages))
4859
print ("width: " + str(self.__somWidth))

‎src/CudaLib/generate_euclidean_distance_matrix_first_step_multi_gpu.h‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -27,11 +27,11 @@ void generate_euclidean_distance_matrix_first_step_multi_gpu(thrust::device_vect
2727
int number_of_threads = omp_get_max_threads();
2828

2929
if (number_of_threads < number_of_gpus) {
30-
std::cout << "Number of threads = " << number_of_threads << std::endl;
31-
std::cout << "Number of GPUs = " << number_of_gpus << std::endl;
32-
std::cout << "GPU IDs = ";
33-
for (auto id : cuda_get_gpu_ids()) std::cout << id << " ";
34-
std::cout << std::endl;
30+
std::cout << "Number of threads = " << number_of_threads << std::endl;
31+
std::cout << "Number of GPUs = " << number_of_gpus << std::endl;
32+
std::cout << "GPU IDs = ";
33+
for (auto id : cuda_get_gpu_ids()) std::cout << id << " ";
34+
std::cout << std::endl;
3535
throw pink::exception("Number of CPU threads must not be smaller than the number of GPU devices");
3636
}
3737

@@ -67,17 +67,17 @@ void generate_euclidean_distance_matrix_first_step_multi_gpu(thrust::device_vect
6767

6868
// Allocate local device memory
6969
if (i != 0) {
70-
thrust::device_vector<EuclideanType> d_som_local(size[i] * neuron_size);
71-
thrust::device_vector<EuclideanType> d_rotated_images_local(number_of_spatial_transformations * neuron_size);
72-
thrust::device_vector<DataType> d_first_step_local(size[i] * number_of_spatial_transformations);
73-
74-
// Copy data
75-
cudaMemcpyPeerAsync(thrust::raw_pointer_cast(d_som_local.data()), i,
76-
thrust::raw_pointer_cast(d_som.data()) + offset[i] * neuron_size, 0,
77-
size[i] * neuron_size * sizeof(EuclideanType), stream);
78-
cudaMemcpyPeerAsync(thrust::raw_pointer_cast(d_rotated_images_local.data()), i,
79-
thrust::raw_pointer_cast(d_rotated_images.data()), 0,
80-
number_of_spatial_transformations * neuron_size * sizeof(EuclideanType), stream);
70+
thrust::device_vector<EuclideanType> d_som_local(size[i] * neuron_size);
71+
thrust::device_vector<EuclideanType> d_rotated_images_local(number_of_spatial_transformations * neuron_size);
72+
thrust::device_vector<DataType> d_first_step_local(size[i] * number_of_spatial_transformations);
73+
74+
// Copy data
75+
cudaMemcpyPeerAsync(thrust::raw_pointer_cast(d_som_local.data()), i,
76+
thrust::raw_pointer_cast(d_som.data()) + offset[i] * neuron_size, 0,
77+
size[i] * neuron_size * sizeof(EuclideanType), stream);
78+
cudaMemcpyPeerAsync(thrust::raw_pointer_cast(d_rotated_images_local.data()), i,
79+
thrust::raw_pointer_cast(d_rotated_images.data()), 0,
80+
number_of_spatial_transformations * neuron_size * sizeof(EuclideanType), stream);
8181

8282
d_som_local_ptr = d_som_local.data();
8383
d_rotated_images_local_ptr = d_rotated_images_local.data();

‎src/Pink/main_generic.h‎

Lines changed: 33 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@ void main_generic(InputData const& input_data)
2828
if (input_data.verbose)
2929
std::cout << "SOM layout: " << SOMLayout::type << "<" << static_cast<int>(SOMLayout::dimensionality) << ">" << "\n"
3030
<< "Data layout: " << DataLayout::type << "<" << static_cast<int>(DataLayout::dimensionality) << ">" << "\n"
31-
<< "GPU usage: " << UseGPU << "\n"
3231
<< std::endl;
3332

3433
SOM<SOMLayout, DataLayout, T> som(input_data);
@@ -93,15 +92,44 @@ void main_generic(InputData const& input_data)
9392
// File for euclidean distances
9493
std::ofstream result_file(input_data.result_filename);
9594
if (!result_file) throw pink::exception("Error opening " + input_data.result_filename);
96-
result_file.write((char*)&input_data.number_of_data_entries, sizeof(int));
97-
som.write_file_header(result_file);
95+
96+
// <file format version> 2 <data-type> <number of entries> <som layout> <data>
97+
int version = 2;
98+
int file_type = 2;
99+
int data_type_idx = 0;
100+
int som_layout_idx = 0;
101+
int som_dimensionality = som.get_som_layout().dimensionality;
102+
int number_of_data_entries = iter_data_cur.get_number_of_entries();
103+
104+
result_file.write((char*)&version, sizeof(int));
105+
result_file.write((char*)&file_type, sizeof(int));
106+
result_file.write((char*)&data_type_idx, sizeof(int));
107+
result_file.write((char*)&number_of_data_entries, sizeof(int));
108+
result_file.write((char*)&som_layout_idx, sizeof(int));
109+
result_file.write((char*)&som_dimensionality, sizeof(int));
110+
for (int dim = 0; dim != som_dimensionality; ++dim) {
111+
int tmp = som.get_som_layout().dimension[dim];
112+
result_file.write((char*)&tmp, sizeof(int));
113+
}
98114

99115
// File for spatial_transformations (optional)
100116
std::ofstream spatial_transformation_file;
101117
if (input_data.write_rot_flip) {
102118
spatial_transformation_file.open(input_data.rot_flip_filename);
103-
spatial_transformation_file.write((char*)&input_data.number_of_data_entries, sizeof(int));
104-
som.write_file_header(spatial_transformation_file);
119+
if (!spatial_transformation_file) throw pink::exception("Error opening " + input_data.rot_flip_filename);
120+
121+
// <file format version> 3 <number of entries> <som layout> <data>
122+
int file_type = 3;
123+
124+
spatial_transformation_file.write((char*)&version, sizeof(int));
125+
spatial_transformation_file.write((char*)&file_type, sizeof(int));
126+
spatial_transformation_file.write((char*)&number_of_data_entries, sizeof(int));
127+
spatial_transformation_file.write((char*)&som_layout_idx, sizeof(int));
128+
spatial_transformation_file.write((char*)&som_dimensionality, sizeof(int));
129+
for (int dim = 0; dim != som_dimensionality; ++dim) {
130+
int tmp = som.get_som_layout().dimension[dim];
131+
spatial_transformation_file.write((char*)&tmp, sizeof(int));
132+
}
105133
}
106134

107135
Mapper_generic<SOMLayout, DataLayout, T, UseGPU> mapper(

‎src/SelfOrganizingMapLib/SOM.cpp‎

Lines changed: 54 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
*/
66

77
#include "SOM.h"
8+
#include "UtilitiesLib/Filler.h"
89

910
namespace pink {
1011

@@ -20,7 +21,59 @@ SOM<CartesianLayout<2>, CartesianLayout<2>, float>::SOM(InputData const& input_d
2021
: som_layout{{input_data.som_width, input_data.som_height}},
2122
neuron_layout{{input_data.neuron_dim, input_data.neuron_dim}},
2223
data(som_layout.size() * neuron_layout.size())
23-
{}
24+
{
25+
// Initialize SOM
26+
if (input_data.init == SOMInitialization::ZERO)
27+
fill_value(&data[0], data.size());
28+
else if (input_data.init == SOMInitialization::RANDOM)
29+
fill_random_uniform(&data[0], data.size(), input_data.seed);
30+
else if (input_data.init == SOMInitialization::RANDOM_WITH_PREFERRED_DIRECTION) {
31+
fill_random_uniform(&data[0], data.size(), input_data.seed);
32+
for (int n = 0; n < input_data.som_size; ++n)
33+
for (uint32_t i = 0; i < input_data.neuron_dim; ++i)
34+
data[n * input_data.neuron_size + i * input_data.neuron_dim + i] = 1.0;
35+
}
36+
else if (input_data.init == SOMInitialization::FILEINIT) {
37+
std::ifstream is(input_data.som_filename);
38+
if (!is) throw pink::exception("Error opening " + input_data.som_filename);
39+
40+
// Skip all header lines starting with #
41+
std::string line;
42+
int last_position = is.tellg();
43+
while (std::getline(is, line)) {
44+
if (line[0] != '#') break;
45+
header += line + '\n';
46+
last_position = is.tellg();
47+
}
48+
49+
is.seekg(last_position, is.beg);
50+
51+
// <file format version> 1 <data-type> <som layout> <neuron layout> <data>
52+
int tmp;
53+
is.read((char*)&tmp, sizeof(int));
54+
if (tmp != 2) throw pink::exception("read SOM: wrong binary file version");
55+
is.read((char*)&tmp, sizeof(int));
56+
if (tmp != 1) throw pink::exception("read SOM: wrong file type");
57+
is.read((char*)&tmp, sizeof(int));
58+
if (tmp != 0) throw pink::exception("read SOM: wrong data type");
59+
is.read((char*)&tmp, sizeof(int));
60+
if (tmp != 0) throw pink::exception("read SOM: wrong SOM layout");
61+
is.read((char*)&tmp, sizeof(int));
62+
if (tmp != som_layout.dimensionality) throw pink::exception("read SOM: wrong SOM dimensionality");
63+
is.read((char*)&tmp, sizeof(int));
64+
if (tmp != static_cast<int>(som_layout.dimension[0])) throw pink::exception("read SOM: wrong SOM dimension[0]");
65+
is.read((char*)&tmp, sizeof(int));
66+
if (tmp != static_cast<int>(som_layout.dimension[0])) throw pink::exception("read SOM: wrong SOM dimension[1]");
67+
is.read((char*)&tmp, sizeof(int));
68+
if (tmp != neuron_layout.dimensionality) throw pink::exception("read SOM: wrong neuron dimensionality");
69+
is.read((char*)&tmp, sizeof(int));
70+
if (tmp != static_cast<int>(neuron_layout.dimension[0])) throw pink::exception("read SOM: wrong neuron dimension[0]");
71+
is.read((char*)&tmp, sizeof(int));
72+
if (tmp != static_cast<int>(neuron_layout.dimension[0])) throw pink::exception("read SOM: wrong neuron dimension[1]");
73+
is.read((char*)&data[0], data.size() * sizeof(float));
74+
} else
75+
throw pink::exception("Unknown SOMInitialization");
76+
}
2477

2578
template <>
2679
SOM<CartesianLayout<3>, CartesianLayout<2>, float>::SOM(InputData const& input_data)

‎src/SelfOrganizingMapLib/SOM.h‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -103,13 +103,6 @@ class SOM
103103
auto get_neuron_dimension() -> typename NeuronLayoutType::DimensionType { return neuron_layout.dimension; }
104104
auto get_neuron_dimension() const -> typename NeuronLayoutType::DimensionType const { return neuron_layout.dimension; }
105105

106-
void write_file_header(std::ofstream& ofs) const
107-
{
108-
int one = 1;
109-
for (uint8_t i = 0; i < som_layout.dimension.size(); ++i) ofs.write((char*)&som_layout.dimension[i], sizeof(int));
110-
for (uint8_t i = som_layout.dimension.size(); i < 3; ++i) ofs.write((char*)&one, sizeof(int));
111-
}
112-
113106
private:
114107

115108
template <typename A, typename B, typename C>

0 commit comments

Comments
 (0)