Skip to content

Commit 15b3785

Browse files
lwshanbdlwshanbd
committed
format
1 parent b3c89a2 commit 15b3785

File tree

2 files changed

+149
-141
lines changed

2 files changed

+149
-141
lines changed

‎llvm_ir_dataset_utils/builders/portage_builder.py

+7-5
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,9 @@ def generate_emerge_command(package_to_build, threads, build_dir):
4545
# This environment variable needs to be set when calling subprocess, not here directly
4646
return command_vector
4747

48-
def perform_build_again(package_name, assembled_build_command, corpus_dir, build_dir):
48+
49+
def perform_build_again(package_name, assembled_build_command, corpus_dir,
50+
build_dir):
4951
logging.info(f"Portage building package {package_name}")
5052
environment = os.environ.copy()
5153
build_log_path = os.path.join(corpus_dir, BUILD_LOG_NAME)
@@ -84,7 +86,8 @@ def perform_build(package_name, assembled_build_command, corpus_dir, build_dir):
8486
logging.warn(f"Failed to build portage package {package_name}")
8587
update_command = ['etc-update', '--automode', '-5']
8688
subprocess.run(update_command)
87-
return perform_build_again(package_name, assembled_build_command, corpus_dir, build_dir)
89+
return perform_build_again(package_name, assembled_build_command,
90+
corpus_dir, build_dir)
8891
logging.info(f"Finished build portage package {package_name}")
8992
return True
9093

@@ -100,7 +103,7 @@ def extract_ir(package_spec, corpus_dir, build_dir, threads):
100103
corpus_dir)
101104
extract_source_lib.copy_source(build_directory, corpus_dir)
102105
return
103-
106+
104107
# Using the tmp directory
105108
build_directory = "/var/tmp/portage/"
106109
package_spec = package_spec + "*"
@@ -157,11 +160,10 @@ def build_package(dependency_futures,
157160
if build_result:
158161
extract_ir(package_spec, corpus_dir, build_dir, threads)
159162
logging.warning(f'Finished building {package_name}')
160-
163+
161164
try:
162165
cleanup(build_dir)
163166
except Exception:
164167
pass
165168

166-
167169
return construct_build_log(build_result, package_name)

‎llvm_ir_dataset_utils/tools/portage_list_build.py

+142-136
Original file line numberDiff line numberDiff line change
@@ -6,159 +6,165 @@
66

77

88
def parse_emerge_output(output):
9-
use_changes = re.findall(r'>=([^\s]+)\s+([^\n]+)', output)
10-
use_changes = [(re.sub(r'-[0-9.]+(?:-r[0-9]+)?$', '', package), flags) for package, flags in use_changes]
11-
return use_changes
9+
use_changes = re.findall(r'>=([^\s]+)\s+([^\n]+)', output)
10+
use_changes = [(re.sub(r'-[0-9.]+(?:-r[0-9]+)?$', '', package), flags)
11+
for package, flags in use_changes]
12+
return use_changes
1213

1314

1415
def update_package_use_custom(use_changes):
15-
package_use_dir = '/etc/portage/package.use'
16-
custom_file_path = os.path.join(package_use_dir, 'custom')
17-
18-
if not os.path.exists(package_use_dir):
19-
os.makedirs(package_use_dir)
20-
21-
if not os.path.exists(custom_file_path):
22-
open(custom_file_path, 'a').close()
23-
24-
with open(custom_file_path, 'r+') as f:
25-
content = f.read()
26-
for package, flags in use_changes:
27-
if package not in content:
28-
f.write(f"{package} {flags}\n")
29-
print(f"Added to {custom_file_path}: {package} {flags}")
30-
else:
31-
print(f"Package {package} already exists in {custom_file_path}")
32-
33-
16+
package_use_dir = '/etc/portage/package.use'
17+
custom_file_path = os.path.join(package_use_dir, 'custom')
18+
19+
if not os.path.exists(package_use_dir):
20+
os.makedirs(package_use_dir)
21+
22+
if not os.path.exists(custom_file_path):
23+
open(custom_file_path, 'a').close()
24+
25+
with open(custom_file_path, 'r+') as f:
26+
content = f.read()
27+
for package, flags in use_changes:
28+
if package not in content:
29+
f.write(f"{package} {flags}\n")
30+
print(f"Added to {custom_file_path}: {package} {flags}")
31+
else:
32+
print(f"Package {package} already exists in {custom_file_path}")
33+
34+
3435
def run_emerge_pretend_again(package_name):
35-
try:
36-
result = subprocess.run(
37-
['emerge', '-pv', '--autounmask-write=y', package_name],
38-
stdout=subprocess.PIPE,
39-
stderr=subprocess.PIPE,
40-
text=True,
41-
check=True
42-
)
43-
return result.stdout
44-
except subprocess.CalledProcessError:
45-
update_command = ['etc-update', '--automode', '-5']
46-
subprocess.run(update_command)
47-
return None
48-
36+
try:
37+
result = subprocess.run(
38+
['emerge', '-pv', '--autounmask-write=y', package_name],
39+
stdout=subprocess.PIPE,
40+
stderr=subprocess.PIPE,
41+
text=True,
42+
check=True)
43+
return result.stdout
44+
except subprocess.CalledProcessError:
45+
update_command = ['etc-update', '--automode', '-5']
46+
subprocess.run(update_command)
47+
return None
48+
49+
4950
def run_emerge_pretend(package_name):
50-
while True:
51-
try:
52-
result = subprocess.run(
53-
['emerge', '-pv', '--autounmask-write=y', package_name],
54-
stdout=subprocess.PIPE,
55-
stderr=subprocess.PIPE,
56-
text=True,
57-
check=True
58-
)
59-
return result.stdout
60-
except subprocess.CalledProcessError as e:
61-
if "have been masked." in e.stderr:
62-
return 1
63-
use_changes = parse_emerge_output(e.stderr)
64-
update_package_use_custom(use_changes)
65-
continue
51+
while True:
52+
try:
53+
result = subprocess.run(
54+
['emerge', '-pv', '--autounmask-write=y', package_name],
55+
stdout=subprocess.PIPE,
56+
stderr=subprocess.PIPE,
57+
text=True,
58+
check=True)
59+
return result.stdout
60+
except subprocess.CalledProcessError as e:
61+
if "have been masked." in e.stderr:
62+
return 1
63+
use_changes = parse_emerge_output(e.stderr)
64+
update_package_use_custom(use_changes)
65+
continue
66+
6667

6768
def extract_package_names(output):
68-
pattern = re.compile(r'(\S+?/\S+?)(?=-\d+(?:\.\d+)*(?:[-:_][a-zA-Z0-9.]+)*)')
69-
matches = pattern.findall(output)
70-
71-
cleaned_matches = []
72-
for match in matches:
73-
if '/' in match and not match.endswith('/'):
74-
cleaned_matches.append(match)
75-
76-
return cleaned_matches
69+
pattern = re.compile(r'(\S+?/\S+?)(?=-\d+(?:\.\d+)*(?:[-:_][a-zA-Z0-9.]+)*)')
70+
matches = pattern.findall(output)
71+
72+
cleaned_matches = []
73+
for match in matches:
74+
if '/' in match and not match.endswith('/'):
75+
cleaned_matches.append(match)
76+
77+
return cleaned_matches
78+
7779

7880
def create_json_file(package):
79-
_ , name = package.split('/')
80-
name_with_slash = package.replace('/', '_')
81-
json_content = {
82-
"sources": [],
83-
"folder_name": name_with_slash,
84-
"build_system": "portage",
85-
"package_name": name,
86-
"package_spec": package
87-
}
81+
_, name = package.split('/')
82+
name_with_slash = package.replace('/', '_')
83+
json_content = {
84+
"sources": [],
85+
"folder_name": name_with_slash,
86+
"build_system": "portage",
87+
"package_name": name,
88+
"package_spec": package
89+
}
8890

89-
json_filename = f"/data/database/json/portage_{name_with_slash}.json"
90-
os.makedirs(os.path.dirname(json_filename), exist_ok=True)
91+
json_filename = f"/data/database/json/portage_{name_with_slash}.json"
92+
os.makedirs(os.path.dirname(json_filename), exist_ok=True)
9193

92-
with open(json_filename, 'w') as json_file:
93-
json.dump(json_content, json_file, indent=2)
94+
with open(json_filename, 'w') as json_file:
95+
json.dump(json_content, json_file, indent=2)
96+
97+
return json_filename
9498

95-
return json_filename
9699

97100
def run_corpus_command(json_filename):
98-
command = [
99-
'python3', './llvm_ir_dataset_utils/tools/corpus_from_description.py',
100-
'--source_dir=/data/database-1/source',
101-
'--corpus_dir=/data/database-1/corpus',
102-
'--build_dir=/data/database-1/build',
103-
f'--corpus_description={json_filename}'
104-
]
105-
try:
106-
subprocess.run(command, check=True)
107-
return 0
108-
except subprocess.CalledProcessError:
109-
return 1
101+
command = [
102+
'python3', './llvm_ir_dataset_utils/tools/corpus_from_description.py',
103+
'--source_dir=/data/database-1/source',
104+
'--corpus_dir=/data/database-1/corpus',
105+
'--build_dir=/data/database-1/build',
106+
f'--corpus_description={json_filename}'
107+
]
108+
try:
109+
subprocess.run(command, check=True)
110+
return 0
111+
except subprocess.CalledProcessError:
112+
return 1
113+
110114

111115
def build(target_package):
112-
installed_packages = []
113-
# Skip finished packages.
114-
with open('../../corpus_descriptions_test/finished.list', 'r') as installed_list:
115-
for i in installed_list:
116-
installed_packages.append(i[:-1])
117-
118-
output = run_emerge_pretend(target_package)
119-
if output == 1:
120-
print(f"{target_package} has been masked")
121-
return
122-
if output:
123-
package_names = extract_package_names(output)
124-
if target_package.replace('/', '_') in installed_packages or \
125-
os.path.exists('/data/database-1/corpus/' + target_package.replace('/', '_')):
126-
return
127-
for package in package_names:
128-
package_use_dir = '/etc/portage/package.use'
129-
custom_file_path = os.path.join(package_use_dir, 'custom')
130-
if os.path.exists(custom_file_path):
131-
os.remove(custom_file_path)
132-
name_with_slash = package.replace('/', '_')
133-
if name_with_slash in installed_packages or os.path.exists('/data/database-1/corpus/' + name_with_slash):
134-
renew_command = ['emerge', '--quiet']
135-
renew_command.append(package)
136-
try:
137-
subprocess.run(renew_command, check=True)
138-
except subprocess.CalledProcessError:
139-
print("Error to build depedency.")
140-
continue
141-
json_filename = create_json_file(package)
142-
run_corpus_command(json_filename)
116+
installed_packages = []
117+
# Skip finished packages.
118+
with open('../../corpus_descriptions_test/finished.list',
119+
'r') as installed_list:
120+
for i in installed_list:
121+
installed_packages.append(i[:-1])
122+
123+
output = run_emerge_pretend(target_package)
124+
if output == 1:
125+
print(f"{target_package} has been masked")
126+
return
127+
if output:
128+
package_names = extract_package_names(output)
129+
if target_package.replace('/', '_') in installed_packages or \
130+
os.path.exists('/data/database-1/corpus/' + target_package.replace('/', '_')):
131+
return
132+
for package in package_names:
133+
package_use_dir = '/etc/portage/package.use'
134+
custom_file_path = os.path.join(package_use_dir, 'custom')
135+
if os.path.exists(custom_file_path):
136+
os.remove(custom_file_path)
137+
name_with_slash = package.replace('/', '_')
138+
if name_with_slash in installed_packages or os.path.exists(
139+
'/data/database-1/corpus/' + name_with_slash):
140+
renew_command = ['emerge', '--quiet']
141+
renew_command.append(package)
142+
try:
143+
subprocess.run(renew_command, check=True)
144+
except subprocess.CalledProcessError:
145+
print("Error to build depedency.")
146+
continue
147+
json_filename = create_json_file(package)
148+
run_corpus_command(json_filename)
143149

144150

145151
if __name__ == "__main__":
146-
if len(sys.argv) != 3:
147-
sys.exit(0)
152+
if len(sys.argv) != 3:
153+
sys.exit(0)
148154

149-
try:
150-
a = int(sys.argv[1])
151-
b = int(sys.argv[2])
152-
except ValueError:
153-
print("Please provide integer values for <a> and <b>.")
154-
sys.exit(1)
155-
156-
with open('../../corpus_descriptions_test/portage_pkg.list', 'r') as file:
157-
for i, package_name in enumerate(file):
158-
if i < a:
159-
continue
160-
if i >= b:
161-
break
162-
package_name = package_name.strip()
163-
package_name = package_name.replace('_','/',1)
164-
build(package_name)
155+
try:
156+
a = int(sys.argv[1])
157+
b = int(sys.argv[2])
158+
except ValueError:
159+
print("Please provide integer values for <a> and <b>.")
160+
sys.exit(1)
161+
162+
with open('../../corpus_descriptions_test/portage_pkg.list', 'r') as file:
163+
for i, package_name in enumerate(file):
164+
if i < a:
165+
continue
166+
if i >= b:
167+
break
168+
package_name = package_name.strip()
169+
package_name = package_name.replace('_', '/', 1)
170+
build(package_name)

0 commit comments

Comments
 (0)