From bdb24be88e9708f880ab1884f1d866e8bdf0b0fe Mon Sep 17 00:00:00 2001 From: Denis 'GNUtoo' Carikli Date: Wed, 15 Apr 2020 15:35:13 +0200 Subject: Start moving into wikidata class Signed-off-by: Denis 'GNUtoo' Carikli --- data/wikidata/cache.py | 2 +- data/wikidata/devices.py | 238 ++-------------------------------------------- data/wikidata/wikidata.py | 170 +++++++++++++++++++++++++++++++++ 3 files changed, 180 insertions(+), 230 deletions(-) create mode 100644 data/wikidata/wikidata.py diff --git a/data/wikidata/cache.py b/data/wikidata/cache.py index 90e2d7a..abb1bd8 100644 --- a/data/wikidata/cache.py +++ b/data/wikidata/cache.py @@ -18,7 +18,7 @@ import json import os import sys -class Cache: +class Cache(object): def __init__(self): file_valid = False diff --git a/data/wikidata/devices.py b/data/wikidata/devices.py index 61483d0..8507650 100755 --- a/data/wikidata/devices.py +++ b/data/wikidata/devices.py @@ -14,222 +14,9 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import os - from optparse import OptionParser -from tabulate import tabulate - -# We don't need a configuration as we only need read-only access to the data -os.environ['PYWIKIBOT_NO_USER_CONFIG'] = '1' -import pywikibot -import pywikibot.config2 - -from cache import * - -# Workaround the following bug: -# https://phabricator.wikimedia.org/T242081 -pywikibot.config2.maxlag = 60 - - -def get_id(data): - # constructs the id from data ['datavalue']['value']['numeric-id'] - value = data.get('datavalue', {}).get('value', {}) - numeric_id = value.get('numeric-id', None) - if numeric_id: - return 'Q' + str(numeric_id) - else: - return None - - -def get_label(data, lang='en'): - return data.get('labels', {}).get(lang, None) - - -def is_a_smartphone_or_tablet_model(page): - smartphone_model_id = 'Q19723451' - tablet_model_id = 'Q155972' - - instances_of = page.get('claims', {}).get('P31', []) - for instance_of in instances_of: - instance_of_target_id = instance_of.getTarget().getID() - if instance_of_target_id == smartphone_model_id: - return True - if instance_of_target_id == tablet_model_id: - return True - return False - - -def get_variant_device_specifications(repo, variant_page): - results = {} - parts = variant_page.get('claims', {}).get('P527', []) - for part in parts: - part_type_id = get_id(part.toJSON().get('mainsnak', {})) - part_model_id = None - - qualifiers = part.toJSON().get('qualifiers', {}).get('P527', []) - assert (len(qualifiers) <= 1), 'Multiple qualifiers' - - for qualifier in qualifiers: - qualifier_id = get_id(qualifier) - if qualifier_id != None: - part_model_id = qualifier_id - part_model_page = pywikibot.ItemPage(repo, part_model_id).get() - part_model_name = get_label(part_model_page) - - if part_type_id == 'Q610398': # System on a chip - results['soc'] = part_model_name; - elif part_type_id == 'Q5607': # Modem - results['modem'] = part_model_name; - elif part_type_id == 'Q5295': # RAM - pass - - return results - - -def get_variant_device_name(variant_page): - variant_superclasses = variant_page.get('claims', {}).get('P279', []) - - for variant_superclass in variant_superclasses: - variant_superclass_page = variant_superclass.getTarget().get() - if is_a_smartphone_or_tablet_model(variant_superclass_page): - model_name = get_label(variant_superclass_page) - return model_name - - return None - - -def get_all_compatible_devices(repo, replicant_page, caching=True): - # Example: {'6.0 0004': { 'GT-I9300': { 'device_name' : 'Galaxy SIII'}}} - results = {} - cache = None - - if caching: - cache = Cache() - #if cache.read(): - results = cache.load() - return results - - replicant_releases_data = replicant_page.get('claims', {}).get('P348', []) - for release_data in replicant_releases_data: - replicant_version = release_data.getTarget() - assert (replicant_version != None), 'replicant_version == None' - - if replicant_version not in results: - results[replicant_version] = {} - release_qualifiers = release_data.toJSON().get('qualifiers', {}) - compatible_variants = release_qualifiers.get('P400', []) - for compatible_variant in compatible_variants: - variant_id = get_id(compatible_variant) - variant_page = pywikibot.ItemPage(repo, variant_id).get() - variant_name = get_label(variant_page) - model_name = get_variant_device_name(variant_page) - assert (variant_name != None), 'variant_name == None' - - if variant_name not in results[replicant_version]: - results[replicant_version][variant_name] = {} - - variant_results = results[replicant_version][variant_name] - - # Enable model_name to be None as it's not essential - if model_name not in results[replicant_version][variant_name]: - variant_results['device_name'] = model_name - - specifications = get_variant_device_specifications(repo, variant_page) - variant_results.update(specifications) - if cache: - cache.store(results) - cache.close() - - return results - - -def get_variant_infos(variant_data, variant): - device_name = variant_data.get('device_name', None) - modem = variant_data.get('modem', None) - soc = variant_data.get('soc', None) - - string = '' - args = [] - - if device_name: - string += '- {} ({})' - args += [device_name, variant] - else: - string += '- {}' - args.append(variant_name) - - if soc: - string += ': {}' - args.append(soc) - - if modem: - args.append(modem) - if soc: - string += ', {}' - else: - string += ': {}' - - return string.format(*args) - -def print_variant_infos_2(variant_data, variant): - device_name = variant_data.get('device_name', None) - modem = variant_data.get('modem', None) - soc = variant_data.get('soc', None) - - headers = [] - table = [] - table.append([]) - - if device_name: - headers.append("Device") - table[0].append(device_name) - - headers.append("Variant") - table[0].append(variant) - else: - headers.append("Variant") - table[0].append(variant_name) - - if soc: - headers.append("SOC") - table[0].append(soc) - - if modem: - headers.append("Modem") - table[0].append(modem) - - print(tabulate(table, headers, tablefmt="psql")) - -def print_compatible_devices(data): - # Example: {'6.0 0004': { 'GT-I9300': { 'device_name' : 'Galaxy SIII'}}} - for replicant_version in data.keys(): - print('Replicant {}:'.format(replicant_version)) - for variant in data[replicant_version].keys(): - variant_data = data[replicant_version][variant] - print(get_variant_infos(variant_data, variant)) - -def print_compatible_variant(data, requested_variant): - # Example: { 'GT-I9300': { 'device_name' : 'Galaxy SIII'}} - for replicant_version in data.keys(): - for variant in data[replicant_version].keys(): - if variant == requested_variant: - variant_data = data[replicant_version][variant] - print_variant_infos_2(variant_data, variant) - return - -def print_device_infos(variant, cache=False): - replicant_releases_data = None - if not cache: - wikidata = pywikibot.Site('wikidata', 'wikidata') - repo = wikidata.data_repository() - replicant_id = 'Q7314062' - replicant_page = pywikibot.ItemPage(repo, replicant_id).get() - replicant_releases_data = get_all_compatible_devices(repo, replicant_page) - else: - replicant_releases_data = get_all_compatible_devices(None, None) - - print_compatible_variant(replicant_releases_data, variant) +from wikidata import * if __name__ == '__main__': parser = OptionParser() @@ -249,27 +36,20 @@ if __name__ == '__main__': cache = parser_options.get('cache', False) if parser_options.get('match_token', None): - # TODO: handle multiple constraints + variants = [] for match_token in options.match_token: # TODO: check format (key, value) = match_token.split('=') if key == 'device': - # TODO: sanity check on value? - print_device_infos(value, cache) + wikidata = Wikidata(cache) + variants.append(value) pass else: #TODO pass + if len (variants) > 0: + wikidata.print_compatible_variants(variants) + else: - replicant_releases_data = None - if not cache: - wikidata = pywikibot.Site('wikidata', 'wikidata') - repo = wikidata.data_repository() - replicant_id = 'Q7314062' - replicant_page = pywikibot.ItemPage(repo, replicant_id).get() - replicant_releases_data = get_all_compatible_devices(repo, replicant_page) - else: - replicant_releases_data = get_all_compatible_devices(None, None) - - print_compatible_devices(replicant_releases_data) - + wikidata = Wikidata(cache) + wikidata.print_compatible_devices() diff --git a/data/wikidata/wikidata.py b/data/wikidata/wikidata.py new file mode 100644 index 0000000..55ed496 --- /dev/null +++ b/data/wikidata/wikidata.py @@ -0,0 +1,170 @@ +#!/bin/env python +# Copyright (C) 2020 Denis 'GNUtoo' Carikli +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import os + +# We don't need a configuration as we only need read-only access to the data +os.environ['PYWIKIBOT_NO_USER_CONFIG'] = '1' +import pywikibot +import pywikibot.config2 + +# Workaround the following bug: +# https://phabricator.wikimedia.org/T242081 +pywikibot.config2.maxlag = 60 + +from tabulate import tabulate + +from cache import * + +class Wikidata(object): + def __init__(self, cache=False): + self.cache = cache + if not self.cache: + site = pywikibot.Site('wikidata', 'wikidata') + self.repo = site.data_repository() + replicant_id = 'Q7314062' + self.replicant_page = pywikibot.ItemPage(self.repo, replicant_id).get() + self.replicant_releases_data = self.get_all_compatible_devices() + + def print_variants_infos(self, variants): + def append_header(headers, text): + if text not in headers: + headers.append(text) + + idx = 0 + headers = [] + table = [] + for variant, variant_data in variants.items(): + device_name = variant_data.get('device_name', None) + modem = variant_data.get('modem', None) + soc = variant_data.get('soc', None) + + table.append([]) + + if device_name: + append_header(headers, "Device") + table[idx].append(device_name) + + append_header(headers, "Variant") + table[idx].append(variant) + else: + append_header(headers, "Variant") + table[idx].append(variant_name) + + if soc: + append_header(headers, "SOC") + table[idx].append(soc) + + if modem: + append_header(headers, "Modem") + table[idx].append(modem) + + idx += 1 + + print(tabulate(table, headers, tablefmt="psql")) + + def print_compatible_variants(self, requested_variants): + # Example: { 'GT-I9300': { 'device_name' : 'Galaxy SIII'}} + results = {} + data = self.replicant_releases_data + + for replicant_version in data.keys(): + for variant in data[replicant_version].keys(): + if variant in requested_variants: + + results[variant] = data[replicant_version][variant] + self.print_variants_infos(results) + + def get_variant_infos(self, variant_data, variant): + device_name = variant_data.get('device_name', None) + modem = variant_data.get('modem', None) + soc = variant_data.get('soc', None) + + string = '' + args = [] + + if device_name: + string += '- {} ({})' + args += [device_name, variant] + else: + string += '- {}' + args.append(variant_name) + + if soc: + string += ': {}' + args.append(soc) + + if modem: + args.append(modem) + if soc: + string += ', {}' + else: + string += ': {}' + + return string.format(*args) + + def get_all_compatible_devices(self): + # Example: {'6.0 0004': { 'GT-I9300': { 'device_name' : 'Galaxy SIII'}}} + results = {} + + if self.cache: + cache = Cache() + #if cache.read(): + results = cache.load() + return results + + replicant_releases_data = self.replicant_page.get('claims', {}).get('P348', []) + for release_data in replicant_releases_data: + replicant_version = release_data.getTarget() + assert (replicant_version != None), 'replicant_version == None' + + if replicant_version not in results: + results[replicant_version] = {} + + release_qualifiers = release_data.toJSON().get('qualifiers', {}) + compatible_variants = release_qualifiers.get('P400', []) + for compatible_variant in compatible_variants: + variant_id = get_id(compatible_variant) + variant_page = pywikibot.ItemPage(self.repo, variant_id).get() + variant_name = get_label(variant_page) + model_name = get_variant_device_name(variant_page) + assert (variant_name != None), 'variant_name == None' + + if variant_name not in results[replicant_version]: + results[replicant_version][variant_name] = {} + + variant_results = results[replicant_version][variant_name] + + # Enable model_name to be None as it's not essential + if model_name not in results[replicant_version][variant_name]: + variant_results['device_name'] = model_name + + specifications = get_variant_device_specifications(self.repo, variant_page) + variant_results.update(specifications) + if self.cache: + cache.store(results) + cache.close() + + return results + + def print_compatible_devices(self): + # Example: {'6.0 0004': { 'GT-I9300': { 'device_name' : 'Galaxy SIII'}}} + data = self.replicant_releases_data + for replicant_version in data.keys(): + print('Replicant {}:'.format(replicant_version)) + for variant in data[replicant_version].keys(): + variant_data = data[replicant_version][variant] + print(self.get_variant_infos(variant_data, variant)) -- cgit v1.2.3