1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
|
\documentclass[aspectratio=169]{beamer}
\mode<presentation> {
\usetheme{Madrid}
\setbeamertemplate{authors}{}
% Remove navigation symbols.
\setbeamertemplate{navigation symbols}{}
}
\usepackage{color}
\usepackage{listings}
\graphicspath{{img/}}
% Use symbols instead of numerals for footnotes.
\renewcommand{\thefootnote}{\fnsymbol{footnote}}
% Reset footnote counter every section.
\makeatletter
\@addtoreset{footnote}{subsection}
\makeatother
\newcommand{\iconframe}[2] {
\begin{minipage}[t]{2.5em}
\includegraphics[width=\textwidth]{app-icons/#1}
\captionof*{figure}{#2}
\end{minipage}
}
\title[Android's build system is messier than your distro's]{Android's build system is messier than your distro's}
\author{Denis `GNUtoo' Carikli \and Ricardo `Grim' Cabrita}
\institute[]{Replicant}
\date{}
\begin{document}
% # Android's build system is messier than your distro's
% The first step in the development of an Android distribution is coming to terms
% with the Android Open Source Project (AOSP) peculiar build system. Despite
% modular, AOSP's build system is quite complex and unlike those used in most
% other distributions. For instance, it lacks the concept of packages and expects
% sub-projects to be explicitly made compatible with it.
\begin{frame}
\maketitle
\end{frame}
\lstset{
basicstyle=\scriptsize\ttfamily,
keywordstyle=\color{blue},
stringstyle=\color{orange},
breaklines=true,
}
% ## AOSP build system internals
% Until version 7 AOSP used a Makefile based system which, in contrast to other
% Make based build systems, didn't rely on recursive Makefiles that find and
% invoke other Makefiles contained in subdirectories. Rather, it used a custom
% script that explores all subdirectories in the source directory in search of
% Android.mk files, which describe how a local module is built and handled. The
% configuration of this system also differed from others such as GNU autotools or
% kernel-style menuconfig, relying instead on variables that are set either as
% part of the shell environment or statically and ahead of time on product and
% device Makefiles.
% As AOSP got larger this system became slow, error prone and hard to scale and
% test. To fix this, the Makefile-based system is getting replaced by Soong. Soong
% replaces Makefiles with Blueprint files, simpler declarative JSON-like
% descriptions of modules, that get translated into a Ninja manifest describing
% the commands that need to be run. This approach takes advantage of Ninja's
% outstanding performance while sparing developers from it's complex build
% language. During the transition period the remaining Makefiles are translated
% to Ninja by Kati instead of being executed directly.
% The transition to Soong solved the performance issue of AOSP's builds but got
% short of taking it closer to the modularity and flexibility found in most
% GNU/Linux distributions.
\section{BuildSystemIntro}
\begin{frame}[fragile]
\frametitle{The Build System}
Before {\bf Android 7} (2006-2016)
\begin{itemize}
\item Makefile based
\item Finds and joins all {\bf Android.mk} files in one
\item Limited configuration during build
\item Error prone
\item Slow and unscalable
\end{itemize}
\end{frame}
\begin{frame}[fragile]
\frametitle{The Build System}
{\bf Android.mk} file example
\begin{block}{}
\lstinputlisting{txt/android-mk-example}
\end{block}
\end{frame}
\begin{frame}[fragile]
\frametitle{The Build System}
From {\bf Android 7} onwards (2016-Now)
\begin{itemize}
\item {\bf Soong Build System}
\item {\bf Blueprint files} are replacing the old {\bf Makefiles}
\item {\bf Blueprint files} are converted to a {\bf Ninja} manifest
\item Remaining {\bf Makefiles} get translated to {\bf Ninja} by {\bf Kati}
\end{itemize}
\end{frame}
\begin{frame}[fragile]
\frametitle{The Build System}
\begin{itemize}
\item {\bf Blueprint files} are simple to write
\item {\bf Ninja} is fast
\item Addresses performance issues
\item Falls short of addressing the lack of modulatiy and flexibility
\end{itemize}
\end{frame}
\begin{frame}[fragile]
\frametitle{The Build System}
{\bf Android.bp} file example
\begin{block}{}
\lstinputlisting{txt/android-bp-example}
\end{block}
\end{frame}
\begin{frame}[fragile]
\frametitle{The Build System}
{\bf build.ninja} file example
\begin{block}{}
\lstinputlisting{txt/ninja-example}
\end{block}
\end{frame}
\begin{frame}[fragile]
\frametitle{Workflow}
\framesubtitle{Getting the source tree}
Initialize the repository
\begin{block}{}
\begin{lstlisting}
repo init -u https://git.replicant.us/replicant/manifest.git -b replicant-9
\end{lstlisting}
\end{block}
\end{frame}
\begin{frame}[fragile]
\frametitle{Workflow}
\framesubtitle{Getting the source tree}
Inside a manifest file
\begin{block}{}
\lstinputlisting{txt/manifest-example}
\end{block}
\end{frame}
\begin{frame}[fragile]
\frametitle{Workflow}
\framesubtitle{Getting the source tree}
Download the source tree
\begin{block}{}
\begin{lstlisting}
repo sync
\end{lstlisting}
\end{block}
Contemplate disk usage
\begin{block}{}
\begin{lstlisting}
du -h --summarize
124GiB
\end{lstlisting}
\end{block}
\end{frame}
\begin{frame}[fragile]
\frametitle{Workflow}
\framesubtitle{Configuring the build}
Initialize the build environment
\begin{block}{}
\begin{lstlisting}
source build/envsetup.sh
\end{lstlisting}
\end{block}
\begin{block}{}
\lstinputlisting{txt/envsetup-commands}
\end{block}
\end{frame}
\begin{frame}[fragile]
\frametitle{Workflow}
\framesubtitle{Configuring the build}
Pick your menu
\begin{block}{}
\begin{lstlisting}
lunch replicant_i9305-userdebug
\end{lstlisting}
\end{block}
\end{frame}
\begin{frame}[fragile]
\frametitle{Workflow}
\framesubtitle{Build}
Start cooking
\begin{block}{}
\begin{lstlisting}
m bacon
\end{lstlisting}
\end{block}
\end{frame}
\begin{frame}[fragile]
\frametitle{Workflow}
\framesubtitle{Upload to device}
\begin{block}{}
\begin{lstlisting}
ls out/target/product/i9305
\end{lstlisting}
\end{block}
\begin{block}{}
\lstinputlisting{txt/output-folder-example}
\end{block}
\end{frame}
\begin{frame}[fragile]
\frametitle{Workflow}
\framesubtitle{Upload to device}
Flash the images directly through download mode
\begin{block}{}
\begin{lstlisting}
heimdall flash --BOOT boot.img --SYSTEM system.img
\end{lstlisting}
\end{block}
Or install through recovery mode
\begin{block}{}
\begin{lstlisting}
adb sideload replicant-ota-eng.user.zip
\end{lstlisting}
\end{block}
\end{frame}
\begin{frame}[fragile]
\frametitle{Workflow}
\framesubtitle{Modify a module}
\begin{block}{}
\begin{lstlisting}
cd external/mesa3d
vim src/gallium/drivers/llvmpipe/Android.mk
mma
m snod
heimdall flash --SYSTEM system.img
\end{lstlisting}
\end{block}
\end{frame}
% ## Building AOSP vs building GNU/Linux distros
% Virtually all GNU/Linux distributions use package managers, which wrap around
% the build system of choice of the software being packaged and bundle it into a
% reusable package that can be installed for direct use and or as a dependency for
% others. These packages can also be delivered and updated via software
% repositories after the initial installation of the distribution. In AOSP, there
% is no package manager. Most software components are built from git repositories
% and bundled into a single image that includes the majority of the distribution.
% Updating a single component thus becomes much harder.
% Besides the core components of AOSP, community distributions tend to use several
% external projects like Mesa and the Linux kernel. These must usually be
% integrated into the build tree as they depend on AOSP's libc. In such cases,
% Google expects the project maintainers to keep their project compatible with
% AOSP's build system, i.e. nowadays they expect the project to have its own
% Blueprint files.
% This contrasts with GNU/Linux distributions, which provide ways to build each
% project in its own native build system and then incorporate it into the distro
% image as an independent package.
\section{ComparisonToOtherDistros}
\begin{frame}[fragile]
\frametitle{Comparison to other distros}
\begin{itemize}
\item Recipe maintenance
\item Software delivery
\end{itemize}
\end{frame}
\begin{frame}[fragile]
\frametitle{Comparison to other distros}
\framesubtitle{Recipe maintenance}
{\bf GNU/Linux}: Most distros use package managers
\begin{block}{}
\lstinputlisting{txt/pkgbuild-example}
\end{block}
\end{frame}
\begin{frame}[fragile]
\frametitle{Comparison to other distros}
\framesubtitle{Recipe maintenance}
{\bf Android}: Projects are expected to keep their own {\bf Android.mk}
\includegraphics[width=\linewidth,height=\textheight,keepaspectratio]{mesa-androidmk}
\end{frame}
\begin{frame}[fragile]
\frametitle{Comparison to other distros}
\framesubtitle{Recipe maintenance}
\begin{itemize}
\item Control of the build is out of the hands of the distro
\item Troublesome to modify build recipes
\item Disregard for the project's prefered build system
\item Changes to the recipe have to be made upstream least the distro wishes to maintain yet another fork
\item Recipe needs to account for multiple Android versions and distros
\item No clear license definition
\end{itemize}
\end{frame}
\begin{frame}[fragile]
\frametitle{Comparison to other distros}
\framesubtitle{Software delivery}
{\bf GNU/Linux}: Packages are delivered through software repositories
\begin{block}{}
\lstinputlisting{txt/gnulinux-packages}
\end{block}
\end{frame}
\begin{frame}[fragile]
\frametitle{Comparison to other distros}
\framesubtitle{Software delivery}
{\bf Android}: Entire system is delivered through a single package
\begin{block}{}
\lstinputlisting{txt/android-single-package}
\end{block}
\end{frame}
\begin{frame}[fragile]
\frametitle{Comparison to other distros}
\framesubtitle{Software delivery}
\begin{itemize}
\item Only developers can update a single component
\item User has to download the entire system for every update
\item Updates are rarely delivered in a timely manner
\item Best case scenario is the OTA update
\end{itemize}
\end{frame}
% ## Replicant's stake
% Replicant, a fully free-software AOSP distribution, is now undergoing intense
% work to make it as sustainable as possible by sharing code with GNU/Linux
% distributions. For instance, Replicant's next release (Replicant 9) is being
% developed on top of mainline Linux and Mesa.
% This comes with the disadvantage of having to deal with the unsuitability of AOSP
% build system to handle external projects. Long hours of development time are
% lost due to the incapability of AOSP to detect changes in projects that haven't
% been converted to Blueprint, such as Linux, Mesa or SwiftShader.
% Replicant is thus seeking to discuss and ultimately decide on a sustainable way
% to go deal with external software components.
% Supporting Blueprint could be advantageous for Replicant, due to its faster
% build times, selective recompilation when source files change on disk and
% ability to support multiple Android versions without convoluted conditional
% cases. However it would be preferable if Android supported some form of packages
% similarly to GNU/Linux distributions. Adding and maintaining Blueprint support
% in all external software components can be incredibly time consuming and could
% even be unsustainable.
\section{Replicant}
\begin{frame}[fragile]
\frametitle{Replicant's stake}
\begin{columns}
\begin{column}{0.2\textwidth}
\includegraphics[width=\textwidth]{replicant}
\end{column}
\begin{column}{0.7\textwidth}
Fully free Android distribution running on several devices\\\bigskip
Undergoing intense work to make it as {\bf sustainable} as possible\\\bigskip
Spending a lot of {\bf development time} on the build system's quirks\\\bigskip
Seeking to discuss on a sustainable way to deal with {\bf external software components}
\end{column}
\end{columns}
\end{frame}
\begin{frame}[fragile]
\frametitle{Replicant's stake}
Arguments for supporting {\bf Soong}
\begin{itemize}
\item It's fast
\item Supports multiple Android versions without convoluted conditional cases
\item Works well for Android components
\end{itemize}
\end{frame}
\begin{frame}[fragile]
\frametitle{Replicant's stake}
Arguments for supporting a {\bf package system}
\begin{itemize}
\item More control on the side of the distro
\item Less forks to maintain
\item More sustainable
\end{itemize}
\end{frame}
% ## End
\section{Ending}
\begin{frame}[fragile]
\frametitle{}
\centerline{Thank you for listening}
\centerline{Come talk to us at the \href{https://36c3.c3nav.de/l/replicant/details/@0,260.22,385.89,-0.56}{\bf Replicant assembly} @ CDC}
\centerline{\url{https://replicant.us}}
\end{frame}
\end{document}
|