From 1e3d584870e7dfb6283f809a2ddc939cd7d7a214 Mon Sep 17 00:00:00 2001 From: Zhaolong Date: Wed, 26 Jan 2022 17:11:24 +0800 Subject: [PATCH] feat: faker data: identification --- Pipfile | 2 + Pipfile.lock | 98 ++++++++++++++++++++++++++++++++++++++++++- app.py | 17 ++++---- base/database.py | 2 +- config.py | 7 +++- model/__init__.py | 3 +- model/person.py | 33 +++++++++++++++ service/faker_data.py | 52 ++++++++++++++++------- utils/model.py | 17 ++++++++ utils/response.py | 3 +- 10 files changed, 206 insertions(+), 28 deletions(-) create mode 100644 utils/model.py diff --git a/Pipfile b/Pipfile index 4a98e83..83aec20 100644 --- a/Pipfile +++ b/Pipfile @@ -29,6 +29,8 @@ MarkupSafe = "==2.0.1" Werkzeug = "==2.0.2" sqlalchemy = "*" flask-sqlalchemy = "*" +pymysql = "*" +cryptography = "*" [dev-packages] diff --git a/Pipfile.lock b/Pipfile.lock index 2038fe0..5e47541 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "122c14e2256186051a240c770381698bc89ad9a43b9eae1532a1d1fa7a6db961" + "sha256": "8f60d328de801415c331d15d1df414d97c273a3c70eac47aec0392631a0b2102" }, "pipfile-spec": 6, "requires": { @@ -24,6 +24,61 @@ "index": "pypi", "version": "==2021.10.8" }, + "cffi": { + "hashes": [ + "sha256:00c878c90cb53ccfaae6b8bc18ad05d2036553e6d9d1d9dbcf323bbe83854ca3", + "sha256:0104fb5ae2391d46a4cb082abdd5c69ea4eab79d8d44eaaf79f1b1fd806ee4c2", + "sha256:06c48159c1abed75c2e721b1715c379fa3200c7784271b3c46df01383b593636", + "sha256:0808014eb713677ec1292301ea4c81ad277b6cdf2fdd90fd540af98c0b101d20", + "sha256:10dffb601ccfb65262a27233ac273d552ddc4d8ae1bf93b21c94b8511bffe728", + "sha256:14cd121ea63ecdae71efa69c15c5543a4b5fbcd0bbe2aad864baca0063cecf27", + "sha256:17771976e82e9f94976180f76468546834d22a7cc404b17c22df2a2c81db0c66", + "sha256:181dee03b1170ff1969489acf1c26533710231c58f95534e3edac87fff06c443", + "sha256:23cfe892bd5dd8941608f93348c0737e369e51c100d03718f108bf1add7bd6d0", + "sha256:263cc3d821c4ab2213cbe8cd8b355a7f72a8324577dc865ef98487c1aeee2bc7", + "sha256:2756c88cbb94231c7a147402476be2c4df2f6078099a6f4a480d239a8817ae39", + "sha256:27c219baf94952ae9d50ec19651a687b826792055353d07648a5695413e0c605", + "sha256:2a23af14f408d53d5e6cd4e3d9a24ff9e05906ad574822a10563efcef137979a", + "sha256:31fb708d9d7c3f49a60f04cf5b119aeefe5644daba1cd2a0fe389b674fd1de37", + "sha256:3415c89f9204ee60cd09b235810be700e993e343a408693e80ce7f6a40108029", + "sha256:3773c4d81e6e818df2efbc7dd77325ca0dcb688116050fb2b3011218eda36139", + "sha256:3b96a311ac60a3f6be21d2572e46ce67f09abcf4d09344c49274eb9e0bf345fc", + "sha256:3f7d084648d77af029acb79a0ff49a0ad7e9d09057a9bf46596dac9514dc07df", + "sha256:41d45de54cd277a7878919867c0f08b0cf817605e4eb94093e7516505d3c8d14", + "sha256:4238e6dab5d6a8ba812de994bbb0a79bddbdf80994e4ce802b6f6f3142fcc880", + "sha256:45db3a33139e9c8f7c09234b5784a5e33d31fd6907800b316decad50af323ff2", + "sha256:45e8636704eacc432a206ac7345a5d3d2c62d95a507ec70d62f23cd91770482a", + "sha256:4958391dbd6249d7ad855b9ca88fae690783a6be9e86df65865058ed81fc860e", + "sha256:4a306fa632e8f0928956a41fa8e1d6243c71e7eb59ffbd165fc0b41e316b2474", + "sha256:57e9ac9ccc3101fac9d6014fba037473e4358ef4e89f8e181f8951a2c0162024", + "sha256:59888172256cac5629e60e72e86598027aca6bf01fa2465bdb676d37636573e8", + "sha256:5e069f72d497312b24fcc02073d70cb989045d1c91cbd53979366077959933e0", + "sha256:64d4ec9f448dfe041705426000cc13e34e6e5bb13736e9fd62e34a0b0c41566e", + "sha256:6dc2737a3674b3e344847c8686cf29e500584ccad76204efea14f451d4cc669a", + "sha256:74fdfdbfdc48d3f47148976f49fab3251e550a8720bebc99bf1483f5bfb5db3e", + "sha256:75e4024375654472cc27e91cbe9eaa08567f7fbdf822638be2814ce059f58032", + "sha256:786902fb9ba7433aae840e0ed609f45c7bcd4e225ebb9c753aa39725bb3e6ad6", + "sha256:8b6c2ea03845c9f501ed1313e78de148cd3f6cad741a75d43a29b43da27f2e1e", + "sha256:91d77d2a782be4274da750752bb1650a97bfd8f291022b379bb8e01c66b4e96b", + "sha256:91ec59c33514b7c7559a6acda53bbfe1b283949c34fe7440bcf917f96ac0723e", + "sha256:920f0d66a896c2d99f0adbb391f990a84091179542c205fa53ce5787aff87954", + "sha256:a5263e363c27b653a90078143adb3d076c1a748ec9ecc78ea2fb916f9b861962", + "sha256:abb9a20a72ac4e0fdb50dae135ba5e77880518e742077ced47eb1499e29a443c", + "sha256:c2051981a968d7de9dd2d7b87bcb9c939c74a34626a6e2f8181455dd49ed69e4", + "sha256:c21c9e3896c23007803a875460fb786118f0cdd4434359577ea25eb556e34c55", + "sha256:c2502a1a03b6312837279c8c1bd3ebedf6c12c4228ddbad40912d671ccc8a962", + "sha256:d4d692a89c5cf08a8557fdeb329b82e7bf609aadfaed6c0d79f5a449a3c7c023", + "sha256:da5db4e883f1ce37f55c667e5c0de439df76ac4cb55964655906306918e7363c", + "sha256:e7022a66d9b55e93e1a845d8c9eba2a1bebd4966cd8bfc25d9cd07d515b33fa6", + "sha256:ef1f279350da2c586a69d32fc8733092fd32cc8ac95139a00377841f59a3f8d8", + "sha256:f54a64f8b0c8ff0b64d18aa76675262e1700f3995182267998c31ae974fbc382", + "sha256:f5c7150ad32ba43a07c4479f40241756145a1f03b43480e058cfd862bf5041c7", + "sha256:f6f824dc3bce0edab5f427efcfb1d63ee75b6fcb7282900ccaf925be84efb0fc", + "sha256:fd8a250edc26254fe5b33be00402e6d287f562b6a5b2152dec302fa15bb3e997", + "sha256:ffaa5c925128e29efbde7301d8ecaf35c8c60ffbcd6a1ffd3a552177c8e5e796" + ], + "version": "==1.15.0" + }, "charset-normalizer": { "hashes": [ "sha256:876d180e9d7432c5d1dfd4c5d26b72f099d503e8fcc0feb7532c9289be60fcbd", @@ -48,6 +103,32 @@ "index": "pypi", "version": "==0.4.4" }, + "cryptography": { + "hashes": [ + "sha256:0a817b961b46894c5ca8a66b599c745b9a3d9f822725221f0e0fe49dc043a3a3", + "sha256:2d87cdcb378d3cfed944dac30596da1968f88fb96d7fc34fdae30a99054b2e31", + "sha256:30ee1eb3ebe1644d1c3f183d115a8c04e4e603ed6ce8e394ed39eea4a98469ac", + "sha256:391432971a66cfaf94b21c24ab465a4cc3e8bf4a939c1ca5c3e3a6e0abebdbcf", + "sha256:39bdf8e70eee6b1c7b289ec6e5d84d49a6bfa11f8b8646b5b3dfe41219153316", + "sha256:4caa4b893d8fad33cf1964d3e51842cd78ba87401ab1d2e44556826df849a8ca", + "sha256:53e5c1dc3d7a953de055d77bef2ff607ceef7a2aac0353b5d630ab67f7423638", + "sha256:596f3cd67e1b950bc372c33f1a28a0692080625592ea6392987dba7f09f17a94", + "sha256:5d59a9d55027a8b88fd9fd2826c4392bd487d74bf628bb9d39beecc62a644c12", + "sha256:6c0c021f35b421ebf5976abf2daacc47e235f8b6082d3396a2fe3ccd537ab173", + "sha256:73bc2d3f2444bcfeac67dd130ff2ea598ea5f20b40e36d19821b4df8c9c5037b", + "sha256:74d6c7e80609c0f4c2434b97b80c7f8fdfaa072ca4baab7e239a15d6d70ed73a", + "sha256:7be0eec337359c155df191d6ae00a5e8bbb63933883f4f5dffc439dac5348c3f", + "sha256:94ae132f0e40fe48f310bba63f477f14a43116f05ddb69d6fa31e93f05848ae2", + "sha256:bb5829d027ff82aa872d76158919045a7c1e91fbf241aec32cb07956e9ebd3c9", + "sha256:ca238ceb7ba0bdf6ce88c1b74a87bffcee5afbfa1e41e173b1ceb095b39add46", + "sha256:ca28641954f767f9822c24e927ad894d45d5a1e501767599647259cbf030b903", + "sha256:e0344c14c9cb89e76eb6a060e67980c9e35b3f36691e15e1b7a9e58a0a6c6dc3", + "sha256:ebc15b1c22e55c4d5566e3ca4db8689470a0ca2babef8e3a9ee057a8b82ce4b1", + "sha256:ec63da4e7e4a5f924b90af42eddf20b698a70e58d86a72d943857c4c6045b3ee" + ], + "index": "pypi", + "version": "==36.0.1" + }, "distlib": { "hashes": [ "sha256:6564fe0a8f51e734df6333d08b8b94d4ea8ee6b99b5ed50613f731fd4089f34b", @@ -338,6 +419,21 @@ "index": "pypi", "version": "==2.4.1" }, + "pycparser": { + "hashes": [ + "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9", + "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206" + ], + "version": "==2.21" + }, + "pymysql": { + "hashes": [ + "sha256:41fc3a0c5013d5f039639442321185532e3e2c8924687abe6537de157d403641", + "sha256:816927a350f38d56072aeca5dfb10221fe1dc653745853d30a216637f5d7ad36" + ], + "index": "pypi", + "version": "==1.0.2" + }, "python-dateutil": { "hashes": [ "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", diff --git a/app.py b/app.py index d7c955c..75e8ca1 100644 --- a/app.py +++ b/app.py @@ -2,25 +2,26 @@ from flask import Flask import config import utils.response as response import service.faker_data as faker -# from base import db -from model import Person +from base import db +from model import Person, SexEnum app = Flask(__name__) -app.config.from_object(config) +config.init_config(app) +db.init_app(app) @app.route('/') def hello_world(): # put application's code here + print(SexEnum(1).sex) return 'Hello World!' @app.route("/faker") def faker_data(): - datas = faker.faker_identifications(2) - print(datas[0]) - print(Person.__dict__.update(datas[0])) - # persons = [Person.__dict__.update(datas[index]) for index in range(len(datas))] - return response.succ({}) + persons = faker.store_faker_identification(1) + results = [person.__to_dict__() for person in persons] + # return response.succ(persons) + return response.succ(faker.faker_train(3)) if __name__ == '__main__': diff --git a/base/database.py b/base/database.py index df4eedf..753c745 100644 --- a/base/database.py +++ b/base/database.py @@ -2,4 +2,4 @@ from flask_sqlalchemy import SQLAlchemy def init_db(app): - db = SQLAlchemy(app) + return SQLAlchemy(app) diff --git a/config.py b/config.py index 8ddcc10..e887ddd 100644 --- a/config.py +++ b/config.py @@ -1,15 +1,18 @@ +from urllib.parse import quote_plus as urlquote + DIALECT = 'mysql' DRIVER = 'pymysql' USERNAME = 'root' -PASSWORD = '1329004986@lm' +PASSWORD = '13291004986@lm' HOST = 'myhost.fallen-angle.com' PORT = '3306' DATABASE = 'ncov_track' -SQLALCHEMY_DATABASE_URI = "{}+{}://{}:{}@{}:{}/{}?charset=utf8".format(DIALECT, DRIVER, USERNAME, PASSWORD, HOST, PORT, DATABASE) +SQLALCHEMY_DATABASE_URI = f'{DIALECT}+{DRIVER}://{USERNAME}:{urlquote(PASSWORD)}@{HOST}:{PORT}/{DATABASE}?charset=utf8' SQLALCHEMY_TRACK_MODIFICATIONS = False SQLALCHEMY_ECHO = True def init_config(app): app.config["SQLALCHEMY_DATABASE_URI"] = SQLALCHEMY_DATABASE_URI + app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False diff --git a/model/__init__.py b/model/__init__.py index 1d35e2a..cfa9376 100644 --- a/model/__init__.py +++ b/model/__init__.py @@ -1 +1,2 @@ -from .person import Person \ No newline at end of file +from .person import Person +from .person import SexEnum \ No newline at end of file diff --git a/model/person.py b/model/person.py index 5e5b86d..2a5d767 100644 --- a/model/person.py +++ b/model/person.py @@ -1,4 +1,5 @@ from base import db +from enum import Enum class Person(db.Model): @@ -9,3 +10,35 @@ class Person(db.Model): age = db.Column(db.Integer, nullable=False) address = db.Column(db.String(20), nullable=False) identification = db.Column(db.String(18), nullable=False) + phone = db.Column(db.String(20), nullable=False) + + @classmethod + def __from_dict__(cls, **data: dict): + obj = cls() + obj.__dict__.update(data) + return obj + + def __to_dict__(self): + return {c.name: str(getattr(self, c.name)) for c in self.__table__.columns} + + +class SexEnum(Enum): + FEMALE = 0, '女' + MALE = 1, '男' + + def __new__(cls, *values): + obj = object.__new__(cls) + # first value is canonical value + obj._value_ = values[0] + for other_value in values[1:]: + cls._value2member_map_[other_value] = obj + obj._all_values = values + obj.sex = values[1] + return obj + + def __repr__(self): + return '<%s.%s: %s>' % ( + self.__class__.__name__, + self._name_, + ', '.join([repr(v) for v in self._all_values]), + ) diff --git a/service/faker_data.py b/service/faker_data.py index b0e39e2..4e6c4e2 100644 --- a/service/faker_data.py +++ b/service/faker_data.py @@ -1,35 +1,59 @@ +import random + import requests import threading +import random from concurrent.futures import ThreadPoolExecutor, as_completed from lxml import etree from faker import Faker +from model import SexEnum, Person +from base import db executor = ThreadPoolExecutor(max_workers=10) faker = Faker(["zh_CN"]) +train_prefix = ['T', 'K', 'D', 'G', 'L', ''] -def faker_identification(): +def faker_identification() -> Person: identification = faker.ssn(max_age=70) resp = requests.get("https://shenfenzheng.bmcx.com/" + identification + "__shenfenzheng/") html = etree.HTML(resp.text) infos = html.xpath('//table[@width="100%"]//tr[position()!=2]/td[@bgcolor="#FFFFFF"]//text()') - result = { - 'identification': infos[0], - 'address': infos[1], - 'sex': infos[3][0], - 'name': faker.last_name() + (faker.first_name_male() if infos[3][0] == '男' else faker.first_name_female()), - 'age': infos[4][0:-2] - } - return result + person = Person( + identification=infos[0], + address=infos[1], + sex=SexEnum(infos[3][0]).value, + name=faker.last_name() + ( + faker.first_name_male() if infos[3][0] == SexEnum.MALE.sex else faker.first_name_female()), + age=infos[4][0:-2], + phone=faker.phone_number() + ) + return person def time(times): return times -def faker_identifications(num: int): +def faker_identifications(num: int) -> list[Person]: all_tasks = [executor.submit(faker_identification) for i in range(num)] - results = [] - for future in as_completed(all_tasks): - results.append(future.result()) - return results + futures = as_completed(all_tasks) + persons = [future.result() for future in futures] + return persons + + +def store_faker_identification(num: int) -> list[Person]: + persons = faker_identifications(num) + db.session.add_all(persons) + db.session.commit() + return persons + + +def faker_train(num: int) -> list[str]: + return [generate_train_id() for i in range(num)] + + +def generate_train_id(): + prefix = train_prefix[int(random.random() * len(train_prefix))] + num = int(random.random() * 10000).__str__() + return prefix + num diff --git a/utils/model.py b/utils/model.py new file mode 100644 index 0000000..bcd507f --- /dev/null +++ b/utils/model.py @@ -0,0 +1,17 @@ +from base import db + + +def model2dict(row: db.Model): + return {c.name: str(getattr(row, c.name)) for c in row.__table__.columns} + + +def models2dicts(rows: list[db.Model]): + return [model2dict(row) for row in rows] + + +def convert2dict(data: object) -> object: + if isinstance(data, list) and len(data) > 0 and isinstance(data[0], db.Model): + data = models2dicts(data) + if isinstance(data, db.Model): + data = model2dict(data) + return data diff --git a/utils/response.py b/utils/response.py index a74a3c3..9b01585 100644 --- a/utils/response.py +++ b/utils/response.py @@ -1,4 +1,4 @@ -from __future__ import annotations +from utils.model import convert2dict def response(resp: dict, status: int, headers=None): @@ -6,6 +6,7 @@ def response(resp: dict, status: int, headers=None): def success(code: int, msg: str, data: object): + data = convert2dict(data) resp = { "code": code, "msg": msg,