1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
|
import argparse
import concurrent.futures
import github
import json
import traceback
from github import Github
from base64 import b64decode
parser = argparse.ArgumentParser()
parser.add_argument('-j', '--jobs', type=int, help='Max number of workers to use. Default is none')
args = parser.parse_args()
with open('token') as f:
g = Github(f.readline().strip(), per_page=200)
print(g.rate_limiting_resettime)
org = g.get_organization('LineageOS')
# supported branches, newest to oldest
CUR_BRANCHES = ['lineage-17.1', 'lineage-16.0', 'lineage-15.1']
def get_cm_dependencies(repo):
branch = None
for b in CUR_BRANCHES:
try:
branch = repo.get_branch(b)
break
except github.GithubException:
continue
if branch is None:
return None
sha = branch.commit.sha
try:
tree = repo.get_git_tree(sha)
except github.GithubException:
return None
blob_sha = None
for el in tree.tree:
if el.path == 'cm.dependencies' or el.path == 'lineage.dependencies':
blob_sha = el.sha
break
if blob_sha is None:
return [[], set()]
blob = repo.get_git_blob(blob_sha)
deps = b64decode(blob.content)
cmdeps = json.loads(deps.decode('utf-8'))
mydeps = []
non_device_repos = set()
for el in cmdeps:
if '_device_' not in el['repository']:
non_device_repos.add(el['repository'])
depbranch = el.get('branch', branch.name)
mydeps.append({'repo': el['repository'], 'branch': depbranch})
return [mydeps, non_device_repos]
futures = {}
n = 1
dependencies = {}
other_repos = set()
with concurrent.futures.ThreadPoolExecutor(max_workers=args.jobs) as executor:
for repo in g.get_organization('LineageOS').get_repos():
if '_device_' not in repo.name:
continue
print(n, repo.name)
n += 1
futures[executor.submit(get_cm_dependencies, repo)] = repo.name
for future in concurrent.futures.as_completed(futures):
name = futures[future]
try:
data = future.result()
if data is None:
continue
dependencies[name] = data[0]
other_repos.update(data[1])
print(name, "=>", data[0])
except Exception as e:
print('%r generated an exception: %s'%(name, e))
traceback.print_exc()
continue
futures = {}
print(other_repos)
for name in other_repos:
print(name)
try:
repo = org.get_repo(name)
futures[executor.submit(get_cm_dependencies, repo)] = name
except Exception:
continue
other_repos = {}
for future in concurrent.futures.as_completed(futures):
name = futures[future]
try:
data = future.result()
if data is None:
continue
dependencies[name] = data[0]
for el in data[1]:
if el in dependencies:
continue
other_repos.update(data[1])
print(name, "=>", data[0])
except Exception as e:
print('%r generated an exception: %s'%(name, e))
traceback.print_exc()
continue
futures = {}
print(other_repos)
#for name in other_repos:
# repo = org.get_repo(name)
# dependencies[name] = get_cm_dependencies(repo)
with open('out.json', 'w') as f:
json.dump(dependencies, f, indent=4)
|
