Skip to content

Commit 629edd0

Browse files
author
Ali
committed
Modern build script
1 parent 18f294a commit 629edd0

File tree

1 file changed

+358
-0
lines changed

1 file changed

+358
-0
lines changed

‎buildbox/build.py

+358
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,358 @@
1+
#!/usr/bin/python3
2+
3+
import argparse
4+
import json
5+
import os
6+
import sys
7+
import shlex
8+
import shutil
9+
import subprocess
10+
import time
11+
12+
from darwin_containers import DarwinContainers
13+
14+
def get_clean_env():
15+
clean_env = os.environ.copy()
16+
return clean_env
17+
18+
def resolve_executable(program):
19+
def is_executable(fpath):
20+
return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
21+
22+
for path in get_clean_env()["PATH"].split(os.pathsep):
23+
executable_file = os.path.join(path, program)
24+
if is_executable(executable_file):
25+
return executable_file
26+
return None
27+
28+
29+
def run_executable_with_output(path, arguments):
30+
executable_path = resolve_executable(path)
31+
if executable_path is None:
32+
raise Exception('Could not resolve {} to a valid executable file'.format(path))
33+
34+
process = subprocess.Popen(
35+
[executable_path] + arguments,
36+
stdout=subprocess.PIPE,
37+
stderr=subprocess.STDOUT,
38+
env=get_clean_env()
39+
)
40+
output_data, _ = process.communicate()
41+
output_string = output_data.decode('utf-8')
42+
return output_string
43+
44+
def session_scp_upload(session, source_path, destination_path):
45+
scp_command = 'scp -i {privateKeyPath} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -pr {source_path} containerhost@"{ipAddress}":{destination_path}'.format(
46+
privateKeyPath=session.privateKeyPath,
47+
ipAddress=session.ipAddress,
48+
source_path=shlex.quote(source_path),
49+
destination_path=shlex.quote(destination_path)
50+
)
51+
if os.system(scp_command) != 0:
52+
print('Command {} finished with a non-zero status'.format(scp_command))
53+
54+
def session_ssh(session, command):
55+
ssh_command = 'ssh -i {privateKeyPath} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null containerhost@"{ipAddress}" -o ServerAliveInterval=60 -t "{command}"'.format(
56+
privateKeyPath=session.privateKeyPath,
57+
ipAddress=session.ipAddress,
58+
command=command
59+
)
60+
return os.system(ssh_command)
61+
62+
def remote_build(darwin_containers_host, configuration):
63+
base_dir = os.getcwd()
64+
65+
configuration_path = 'versions.json'
66+
xcode_version = ''
67+
with open(configuration_path) as file:
68+
configuration_dict = json.load(file)
69+
if configuration_dict['xcode'] is None:
70+
raise Exception('Missing xcode version in {}'.format(configuration_path))
71+
xcode_version = configuration_dict['xcode']
72+
73+
print('Xcode version: {}'.format(xcode_version))
74+
75+
commit_count = run_executable_with_output('git', [
76+
'rev-list',
77+
'--count',
78+
'HEAD'
79+
])
80+
81+
build_number_offset = 0
82+
with open('build_number_offset') as file:
83+
build_number_offset = int(file.read())
84+
85+
build_number = build_number_offset + int(commit_count)
86+
print('Build number: {}'.format(build_number))
87+
88+
macos_version = '12.5'
89+
image_name = 'macos-{macos_version}-xcode-{xcode_version}'.format(macos_version=macos_version, xcode_version=xcode_version)
90+
91+
print('Image name: {}'.format(image_name))
92+
93+
buildbox_dir = 'buildbox'
94+
os.makedirs('{buildbox_dir}/transient-data'.format(buildbox_dir=buildbox_dir), exist_ok=True)
95+
96+
codesigning_subpath = ''
97+
remote_configuration = ''
98+
if configuration == 'appcenter':
99+
remote_configuration = 'hockeyapp'
100+
elif configuration == 'appstore':
101+
remote_configuration = 'appstore'
102+
elif configuration == 'reproducible':
103+
codesigning_subpath = 'build-system/fake-codesigning'
104+
remote_configuration = 'verify'
105+
106+
destination_codesigning_path = '{buildbox_dir}/transient-data/telegram-codesigning'.format(buildbox_dir=buildbox_dir)
107+
destination_build_configuration_path = '{buildbox_dir}/transient-data/build-configuration'.format(buildbox_dir=buildbox_dir)
108+
109+
if os.path.exists(destination_codesigning_path):
110+
shutil.rmtree(destination_codesigning_path)
111+
if os.path.exists(destination_build_configuration_path):
112+
shutil.rmtree(destination_build_configuration_path)
113+
114+
shutil.copytree('build-system/fake-codesigning', '{buildbox_dir}/transient-data/telegram-codesigning'.format(buildbox_dir=buildbox_dir))
115+
shutil.copytree('build-system/example-configuration', '{buildbox_dir}/transient-data/build-configuration'.format(buildbox_dir=buildbox_dir))
116+
else:
117+
print('Unknown configuration {}'.format(configuration))
118+
sys.exit(1)
119+
120+
source_dir = os.path.basename(base_dir)
121+
source_archive_path = '{buildbox_dir}/transient-data/source.tar'.format(buildbox_dir=buildbox_dir)
122+
123+
if os.path.exists(source_archive_path):
124+
os.remove(source_archive_path)
125+
126+
print('Compressing source code...')
127+
os.system('find . -type f -a -not -regex "\\." -a -not -regex ".*\\./git" -a -not -regex ".*\\./git/.*" -a -not -regex "\\./bazel-bin" -a -not -regex "\\./bazel-bin/.*" -a -not -regex "\\./bazel-out" -a -not -regex "\\./bazel-out/.*" -a -not -regex "\\./bazel-testlogs" -a -not -regex "\\./bazel-testlogs/.*" -a -not -regex "\\./bazel-telegram-ios" -a -not -regex "\\./bazel-telegram-ios/.*" -a -not -regex "\\./buildbox" -a -not -regex "\\./buildbox/.*" -a -not -regex "\\./buck-out" -a -not -regex "\\./buck-out/.*" -a -not -regex "\\./\\.buckd" -a -not -regex "\\./\\.buckd/.*" -a -not -regex "\\./build" -a -not -regex "\\./build/.*" -print0 | tar cf "{buildbox_dir}/transient-data/source.tar" --null -T -'.format(buildbox_dir=buildbox_dir))
128+
129+
darwinContainers = DarwinContainers(serverAddress=darwin_containers_host, verbose=False)
130+
131+
print('Opening container session...')
132+
with darwinContainers.workingImageSession(name=image_name) as session:
133+
print('Uploading data to container...')
134+
session_scp_upload(session=session, source_path=codesigning_subpath, destination_path='codesigning_data')
135+
session_scp_upload(session=session, source_path='{base_dir}/{buildbox_dir}/transient-data/build-configuration'.format(base_dir=base_dir, buildbox_dir=buildbox_dir), destination_path='telegram-configuration')
136+
session_scp_upload(session=session, source_path='{base_dir}/{buildbox_dir}/guest-build-telegram.sh'.format(base_dir=base_dir, buildbox_dir=buildbox_dir), destination_path='')
137+
session_scp_upload(session=session, source_path='{base_dir}/{buildbox_dir}/transient-data/source.tar'.format(base_dir=base_dir, buildbox_dir=buildbox_dir), destination_path='')
138+
139+
print('Executing remote build...')
140+
141+
bazel_cache_host=''
142+
session_ssh(session=session, command='BUILD_NUMBER="{build_number}" BAZEL_HTTP_CACHE_URL="{bazel_cache_host}" bash -l guest-build-telegram.sh {remote_configuration}'.format(
143+
build_number=build_number,
144+
bazel_cache_host=bazel_cache_host,
145+
remote_configuration=remote_configuration
146+
))
147+
148+
print('Retrieving build artifacts...')
149+
150+
artifacts_path='{base_dir}/build/artifacts'.format(base_dir=base_dir)
151+
if os.path.exists(artifacts_path):
152+
shutil.rmtree(artifacts_path)
153+
os.makedirs(artifacts_path, exist_ok=True)
154+
155+
session_scp_download(session=session, source_path='telegram-ios/build/artifacts/*', destination_path='{artifacts_path}/'.format(artifacts_path=artifacts_path))
156+
print('Artifacts have been stored at {}'.format(artifacts_path))
157+
158+
if __name__ == '__main__':
159+
parser = argparse.ArgumentParser(prog='build')
160+
161+
parser.add_argument(
162+
'--verbose',
163+
action='store_true',
164+
default=False,
165+
help='Print debug info'
166+
)
167+
168+
subparsers = parser.add_subparsers(dest='commandName', help='Commands')
169+
170+
remote_build_parser = subparsers.add_parser('remote-build', help='Build the app using a remote environment.')
171+
remote_build_parser.add_argument(
172+
'--darwinContainersHost',
173+
required=True,
174+
type=str,
175+
help='DarwinContainers host address.'
176+
)
177+
remote_build_parser.add_argument(
178+
'--configuration',
179+
choices=[
180+
'appcenter',
181+
'appstore',
182+
'reproducible'
183+
],
184+
required=True,
185+
help='Build configuration'
186+
)
187+
188+
if len(sys.argv) < 2:
189+
parser.print_help()
190+
sys.exit(1)
191+
192+
args = parser.parse_args()
193+
194+
if args.commandName is None:
195+
exit(0)
196+
197+
if args.commandName == 'remote-build':
198+
remote_build(darwin_containers_host=args.darwinContainersHost, configuration=args.configuration)
199+
200+
201+
'''set -e
202+
203+
rm -f "tools/bazel"
204+
cp "$BAZEL" "tools/bazel"
205+
206+
BUILD_CONFIGURATION="$1"
207+
208+
if [ "$BUILD_CONFIGURATION" == "hockeyapp" ] || [ "$BUILD_CONFIGURATION" == "appcenter-experimental" ] || [ "$BUILD_CONFIGURATION" == "appcenter-experimental-2" ]; then
209+
CODESIGNING_SUBPATH="$BUILDBOX_DIR/transient-data/telegram-codesigning/codesigning"
210+
elif [ "$BUILD_CONFIGURATION" == "appstore" ] || [ "$BUILD_CONFIGURATION" == "appstore-development" ]; then
211+
CODESIGNING_SUBPATH="$BUILDBOX_DIR/transient-data/telegram-codesigning/codesigning"
212+
elif [ "$BUILD_CONFIGURATION" == "verify" ]; then
213+
CODESIGNING_SUBPATH="build-system/fake-codesigning"
214+
else
215+
echo "Unknown configuration $1"
216+
exit 1
217+
fi
218+
219+
COMMIT_COMMENT="$(git log -1 --pretty=%B)"
220+
case "$COMMIT_COMMENT" in
221+
*"[nocache]"*)
222+
export BAZEL_HTTP_CACHE_URL=""
223+
;;
224+
esac
225+
226+
COMMIT_ID="$(git rev-parse HEAD)"
227+
COMMIT_AUTHOR=$(git log -1 --pretty=format:'%an')
228+
if [ -z "$2" ]; then
229+
COMMIT_COUNT=$(git rev-list --count HEAD)
230+
BUILD_NUMBER_OFFSET="$(cat build_number_offset)"
231+
COMMIT_COUNT="$(($COMMIT_COUNT+$BUILD_NUMBER_OFFSET))"
232+
BUILD_NUMBER="$COMMIT_COUNT"
233+
else
234+
BUILD_NUMBER="$2"
235+
fi
236+
237+
BASE_DIR=$(pwd)
238+
239+
if [ "$BUILD_CONFIGURATION" == "hockeyapp" ] || [ "$BUILD_CONFIGURATION" == "appcenter-experimental" ] || [ "$BUILD_CONFIGURATION" == "appcenter-experimental-2" ] || [ "$BUILD_CONFIGURATION" == "appstore" ] || [ "$BUILD_CONFIGURATION" == "appstore-development" ]; then
240+
if [ ! `which generate-configuration.sh` ]; then
241+
echo "generate-configuration.sh not found in PATH $PATH"
242+
exit 1
243+
fi
244+
245+
mkdir -p "$BASE_DIR/$BUILDBOX_DIR/transient-data/telegram-codesigning"
246+
mkdir -p "$BASE_DIR/$BUILDBOX_DIR/transient-data/build-configuration"
247+
248+
case "$BUILD_CONFIGURATION" in
249+
"hockeyapp"|"appcenter-experimental"|"appcenter-experimental-2")
250+
generate-configuration.sh internal release "$BASE_DIR/$BUILDBOX_DIR/transient-data/telegram-codesigning" "$BASE_DIR/$BUILDBOX_DIR/transient-data/build-configuration"
251+
;;
252+
253+
"appstore")
254+
generate-configuration.sh appstore release "$BASE_DIR/$BUILDBOX_DIR/transient-data/telegram-codesigning" "$BASE_DIR/$BUILDBOX_DIR/transient-data/build-configuration"
255+
;;
256+
257+
"appstore-development")
258+
generate-configuration.sh appstore development "$BASE_DIR/$BUILDBOX_DIR/transient-data/telegram-codesigning" "$BASE_DIR/$BUILDBOX_DIR/transient-data/build-configuration"
259+
;;
260+
261+
*)
262+
echo "Unknown build configuration $BUILD_CONFIGURATION"
263+
exit 1
264+
;;
265+
esac
266+
elif [ "$BUILD_CONFIGURATION" == "verify" ]; then
267+
mkdir -p "$BASE_DIR/$BUILDBOX_DIR/transient-data/telegram-codesigning"
268+
mkdir -p "$BASE_DIR/$BUILDBOX_DIR/transient-data/build-configuration"
269+
270+
cp -R build-system/fake-codesigning/* "$BASE_DIR/$BUILDBOX_DIR/transient-data/telegram-codesigning/"
271+
cp -R build-system/example-configuration/* "$BASE_DIR/$BUILDBOX_DIR/transient-data/build-configuration/"
272+
fi
273+
274+
if [ ! -d "$CODESIGNING_SUBPATH" ]; then
275+
echo "$CODESIGNING_SUBPATH does not exist"
276+
exit 1
277+
fi
278+
279+
SOURCE_DIR=$(basename "$BASE_DIR")
280+
rm -f "$BUILDBOX_DIR/transient-data/source.tar"
281+
set -x
282+
find . -type f -a -not -regex "\\." -a -not -regex ".*\\./git" -a -not -regex ".*\\./git/.*" -a -not -regex "\\./bazel-bin" -a -not -regex "\\./bazel-bin/.*" -a -not -regex "\\./bazel-out" -a -not -regex "\\./bazel-out/.*" -a -not -regex "\\./bazel-testlogs" -a -not -regex "\\./bazel-testlogs/.*" -a -not -regex "\\./bazel-telegram-ios" -a -not -regex "\\./bazel-telegram-ios/.*" -a -not -regex "\\./buildbox" -a -not -regex "\\./buildbox/.*" -a -not -regex "\\./buck-out" -a -not -regex "\\./buck-out/.*" -a -not -regex "\\./\\.buckd" -a -not -regex "\\./\\.buckd/.*" -a -not -regex "\\./build" -a -not -regex "\\./build/.*" -print0 | tar cf "$BUILDBOX_DIR/transient-data/source.tar" --null -T -
283+
284+
PROCESS_ID="$$"
285+
286+
if [ -z "$RUNNING_VM" ]; then
287+
VM_NAME="$VM_BASE_NAME-$(openssl rand -hex 10)-build-telegram-$PROCESS_ID"
288+
else
289+
VM_NAME="$RUNNING_VM"
290+
fi
291+
292+
if [ "$BUILD_MACHINE" == "linux" ]; then
293+
virt-clone --original "$VM_BASE_NAME" --name "$VM_NAME" --auto-clone
294+
virsh start "$VM_NAME"
295+
296+
echo "Getting VM IP"
297+
298+
while [ 1 ]; do
299+
TEST_IP=$(virsh domifaddr "$VM_NAME" 2>/dev/null | egrep -o 'ipv4.*' | sed -e 's/ipv4\s*//g' | sed -e 's|/.*||g')
300+
if [ ! -z "$TEST_IP" ]; then
301+
RESPONSE=$(ssh -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null telegram@"$TEST_IP" -o ServerAliveInterval=60 -t "echo -n 1")
302+
if [ "$RESPONSE" == "1" ]; then
303+
VM_IP="$TEST_IP"
304+
break
305+
fi
306+
fi
307+
sleep 1
308+
done
309+
elif [ "$BUILD_MACHINE" == "macOS" ]; then
310+
if [ -z "$RUNNING_VM" ]; then
311+
prlctl clone "$VM_BASE_NAME" --linked --name "$VM_NAME"
312+
prlctl start "$VM_NAME"
313+
314+
echo "Getting VM IP"
315+
316+
while [ 1 ]; do
317+
TEST_IP=$(prlctl exec "$VM_NAME" "ifconfig | grep inet | grep broadcast | grep -Eo '([0-9]{1,3}\.){3}[0-9]{1,3}' | head -1 | tr '\n' '\0'" 2>/dev/null || echo "")
318+
if [ ! -z "$TEST_IP" ]; then
319+
RESPONSE=$(ssh -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null telegram@"$TEST_IP" -o ServerAliveInterval=60 -t "echo -n 1")
320+
if [ "$RESPONSE" == "1" ]; then
321+
VM_IP="$TEST_IP"
322+
break
323+
fi
324+
fi
325+
sleep 1
326+
done
327+
fi
328+
echo "VM_IP=$VM_IP"
329+
fi
330+
331+
scp -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -pr "$CODESIGNING_SUBPATH" telegram@"$VM_IP":codesigning_data
332+
scp -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -pr "$BASE_DIR/$BUILDBOX_DIR/transient-data/build-configuration" telegram@"$VM_IP":telegram-configuration
333+
334+
scp -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -pr "$BUILDBOX_DIR/guest-build-telegram.sh" "$BUILDBOX_DIR/transient-data/source.tar" telegram@"$VM_IP":
335+
336+
ssh -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null telegram@"$VM_IP" -o ServerAliveInterval=60 -t "export BUILD_NUMBER=\"$BUILD_NUMBER\"; export BAZEL_HTTP_CACHE_URL=\"$BAZEL_HTTP_CACHE_URL\"; $GUEST_SHELL -l guest-build-telegram.sh $BUILD_CONFIGURATION" || true
337+
338+
OUTPUT_PATH="build/artifacts"
339+
rm -rf "$OUTPUT_PATH"
340+
mkdir -p "$OUTPUT_PATH"
341+
342+
scp -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -pr telegram@"$VM_IP":"telegram-ios/build/artifacts/*" "$OUTPUT_PATH/"
343+
344+
if [ -z "$RUNNING_VM" ]; then
345+
if [ "$BUILD_MACHINE" == "linux" ]; then
346+
virsh destroy "$VM_NAME"
347+
virsh undefine "$VM_NAME" --remove-all-storage --nvram
348+
elif [ "$BUILD_MACHINE" == "macOS" ]; then
349+
echo "Deleting VM..."
350+
#prlctl stop "$VM_NAME" --kill
351+
#prlctl delete "$VM_NAME"
352+
fi
353+
fi
354+
355+
if [ ! -f "$OUTPUT_PATH/Telegram.ipa" ]; then
356+
exit 1
357+
fi
358+
'''

0 commit comments

Comments
 (0)