diff --git a/.github/workflows/ci-matrix.yml b/.github/workflows/ci-matrix.yml index 8a9282e1e..c150867e5 100644 --- a/.github/workflows/ci-matrix.yml +++ b/.github/workflows/ci-matrix.yml @@ -20,7 +20,7 @@ jobs: name: Build node on ${{ matrix.target.os }} runs-on: ${{ matrix.target.os }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} @@ -36,7 +36,8 @@ jobs: toolchain: 1.63.0 components: rustfmt, clippy override: true - - uses: actions/cache@v3 + + - uses: actions/cache@v4 with: path: | ~/.cargo/bin/ @@ -45,10 +46,26 @@ jobs: ~/.cargo/registry/cache/ ~/.cargo/git/db/ target/ - key: ${{ runner.os }}-cargo + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-cargo- + + - name: Download dbip file + uses: actions/download-artifact@v4 + with: + name: dbip_country + + - name: Move and overwrite dbip file + shell: bash + run: cp -f dbip_country.rs ip_country/src/dbip_country.rs || exit 1 + + - name: Show dbip file + run: cat ip_country/src/dbip_country.rs - name: Build ${{ matrix.target.os }} run: | + git fetch + git checkout origin/generated-source -- ip_country/src/dbip_country.rs ./ci/all.sh ./ci/multinode_integration_test.sh ./ci/collect_results.sh @@ -59,17 +76,23 @@ jobs: with: name: Node-${{ matrix.target.name }} path: results - + - name: diagnostics + if: failure() + run: | + echo "final disc diagnostics ------>" + df -h /Users/runner/work/Node/Node/node/target/release deploy_to_s3: needs: build + if: success() && (startsWith(github.head_ref, 'GH') || startsWith(github.head_ref, 'v')) strategy: matrix: - os: [linux, macos, windows] - runs-on: ubuntu-22.04 + os: [linux, macos, windows] + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 1 - name: Download artifacts uses: actions/download-artifact@v4 @@ -77,31 +100,35 @@ jobs: - name: Display structure of downloaded files run: ls -R + - name: Check artifacts exist + run: | + if [ ! -d "Node-${{ matrix.os }}/generated/bin/" ]; then + echo "Error: Build artifacts not found" + exit 1 + fi + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + - if: startsWith(github.head_ref, 'GH') name: Versioned S3 Sync - uses: jakejarvis/s3-sync-action@v0.5.1 - with: - args: --acl private --follow-symlinks --delete - env: - AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: 'us-west-2' - DEST_DIR: 'Node/${{ github.head_ref }}/Node-${{ matrix.os }}' - SOURCE_DIR: 'Node-${{ matrix.os }}/generated/bin/' + run: | + aws s3 sync "Node-${{ matrix.os }}/generated/bin/" "s3://${{ secrets.AWS_S3_BUCKET }}/Node/${{ github.head_ref }}/Node-${{ matrix.os }}" \ + --delete \ + --no-progress \ + --acl private - if: startsWith(github.head_ref, 'v') name: Latest S3 Sync - uses: jakejarvis/s3-sync-action@v0.5.1 - with: - args: --acl private --follow-symlinks --delete - env: - AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: 'us-west-2' - DEST_DIR: 'Node/latest/Node-${{ matrix.os }}' - SOURCE_DIR: 'Node-${{ matrix.os }}/generated/bin/' + run: | + aws s3 sync "Node-${{ matrix.os }}/generated/bin/" "s3://${{ secrets.AWS_S3_BUCKET }}/Node/latest/Node-${{ matrix.os }}" \ + --delete \ + --no-progress \ + --acl private - name: Invalidate Binaries CloudFront uses: chetan/invalidate-cloudfront-action@v2.4 diff --git a/.github/workflows/dbip_download.yml b/.github/workflows/dbip_download.yml new file mode 100644 index 000000000..843005a68 --- /dev/null +++ b/.github/workflows/dbip_download.yml @@ -0,0 +1,63 @@ +name: Download DBIP data and generate dbip_country.rs + +on: + workflow_dispatch: +# schedule: +# - cron: "0 0 3 * *" # Runs at midnight on the 3rd of every month + +env: + SKIP_EXPORT: 'false' + TEMP_DIR: '/tmp' + YEAR_MONTH: '01-9999' + +permissions: + contents: write + +jobs: + generate: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set up Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + + - name: Download DBIP data in MMDB format + run: | + cd ip_country + export YEAR_MONTH=$(date +%Y-%m) + echo "TEMP_DIR=$(mktemp -d)" >> $GITHUB_ENV + echo "YEAR_MONTH=$YEAR_MONTH" >> $GITHUB_ENV + mkdir -p dbip-data + curl -L -o dbip-data/dbip-country-lite.mmdb.gz "https://download.db-ip.com/free/dbip-country-lite-$YEAR_MONTH.mmdb.gz" + gunzip dbip-data/dbip-country-lite.mmdb.gz + if [ "$?" -ne "0" ]; then + echo "SKIP_EXPORT=true" >> $GITHUB_ENV + fi + + - name: Generate Rust source file + if: ${{ env.SKIP_EXPORT == 'false' }} + run: | + cd ip_country + mkdir -p generated + cargo run < "dbip-data/dbip-country-lite.mmdb" > "$TEMP_DIR"/dbip_country.rs + ls "$TEMP_DIR" + if [ "$?" -ne "0" ]; then + echo "SKIP_EXPORT=true" >> $GITHUB_ENV + fi + + - name: Commit and push generated file + if: ${{ env.SKIP_EXPORT == 'false' }} + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git checkout -B generated-source + git rm -rf . + mkdir ip_country/src + mv "${TEMP_DIR}"/dbip_country.rs ip_country/src/dbip_country.rs + git add ip_country/src/dbip_country.rs + git commit -m "Update generated dbip_country ${YEAR_MONTH} Rust source file" + git push origin generated-source --force \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml index ed9ea5ac4..0d0cdcbb2 100644 --- a/.idea/modules.xml +++ b/.idea/modules.xml @@ -4,11 +4,14 @@ + + + \ No newline at end of file diff --git a/USER-INTERFACE-INTERFACE.md b/USER-INTERFACE-INTERFACE.md index c7059929a..134c6ae64 100644 --- a/USER-INTERFACE-INTERFACE.md +++ b/USER-INTERFACE-INTERFACE.md @@ -590,6 +590,67 @@ descriptor (for example, if it's still waiting on the router to get a public IP Node descriptor (for example, if its neighborhood mode is not Standard), the `nodeDescriptorOpt` field will be null or absent. +#### `exit-location` +##### Direction: Request +##### Correspondent: Node +##### Layout: +``` +"payload": { + "fallbackRouting": , + "exitLocations": [ + { + "countryCodes": [string, ..], + "priority": + }, + ], + "showCountries": +} +``` +##### Description: +This command requests information about the countries available for exit in our neighborhood and allows us to set up the +desired locations with their priority. The priority provides the node's perspective on how important a particular country +is for our preferences. + +This command can be used in two ways which can't be combined: +1. If we use the command with showCountries set to true, it retrieves information about the available countries in our neighborhood. In this case, other parameters are ignored. +2. If we want to set an exit location, we must set showCountries to false and then configure fallbackRouting and exitLocations with our preferences. + +The fallbackRouting parameter determines whether we want to block exit for a particular country. If this country is no longer +available, the route to exit will fail during construction. If fallbackRouting is set to true, we can exit through any available +country if none of our specified exitLocations are accessible. + +Priorities are used to determine the preferred exit countries. Priority 1 is the highest, while higher numbers indicate +lower priority. For example, if we specify DE with priority 1 and FR with priority 2, then an exit through France will +only be used if a German exit is unavailable or significantly more expensive. + +#### `exit-location` +##### Direction: Response +##### Correspondent: UI +##### Layout: + +``` +"payload": { + "fallbackRouting": , + "exitCountrySelection": <[ + { + "countryCodes": [string, ..], + "priority": + }, + ]>, + "missingCountries": <[string, ..]> + "exitCountries": +} +``` +##### Description: +In response, the Node sends a payload to the UI that contains either the Exit Location settings (which may include missing countries) or a list of exit countries. + +Exit Location settings consist of fallbackRouting, exitCountrySelection, and missingCountries, where: +1. fallbackRouting is a boolean representing the user's choice to enable or disable fallback routing within the neighborhood. +2. exitCountrySelection is an array of objects, where each object represents a set of country codes along with their assigned priority. +3. missingCountries is an array of strings representing a list of countries that are currently unavailable in the Node's Neighborhood Database. + +Exit Countries (or exitCountries) is an optional array containing ISO country code strings. These represent the countries currently available in the Node's Neighborhood Database. The user can select from these countries to configure the Exit Location settings. + #### `financials` ##### Direction: Request ##### Correspondent: Node diff --git a/automap/Cargo.lock b/automap/Cargo.lock index 73d051ebb..2ddb2561c 100644 --- a/automap/Cargo.lock +++ b/automap/Cargo.lock @@ -351,8 +351,8 @@ version = "0.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d897f47bf7270cf70d370f8f98c1abb6d2d4cf60a6845d30e05bfb90c6568650" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.26", + "proc-macro2 1.0.94", + "quote 1.0.40", "unicode-xid 0.2.4", ] @@ -464,6 +464,27 @@ dependencies = [ "subtle 1.0.0", ] +[[package]] +name = "csv" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" +dependencies = [ + "memchr", +] + [[package]] name = "cxx" version = "1.0.94" @@ -485,10 +506,10 @@ dependencies = [ "cc", "codespan-reporting", "once_cell", - "proc-macro2 1.0.56", - "quote 1.0.26", + "proc-macro2 1.0.94", + "quote 1.0.40", "scratch", - "syn 2.0.15", + "syn 2.0.100", ] [[package]] @@ -503,9 +524,9 @@ version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2345488264226bf682893e25de0769f3360aac9957980ec49361b083ddaa5bc5" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.26", - "syn 2.0.15", + "proc-macro2 1.0.94", + "quote 1.0.40", + "syn 2.0.100", ] [[package]] @@ -595,8 +616,8 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.26", + "proc-macro2 1.0.94", + "quote 1.0.40", "syn 1.0.109", "synstructure", ] @@ -631,7 +652,7 @@ dependencies = [ "lazy_static", "log 0.4.17", "regex", - "thiserror", + "thiserror 1.0.40", "yansi", ] @@ -901,6 +922,17 @@ dependencies = [ "libc", ] +[[package]] +name = "ip_country" +version = "0.1.0" +dependencies = [ + "csv", + "ipnetwork", + "itertools 0.13.0", + "lazy_static", + "maxminddb", +] + [[package]] name = "ipconfig" version = "0.1.9" @@ -914,6 +946,12 @@ dependencies = [ "winreg", ] +[[package]] +name = "ipnetwork" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf370abdafd54d13e54a620e8c3e1145f28e46cc9d704bc6d94414559df41763" + [[package]] name = "itertools" version = "0.10.5" @@ -923,6 +961,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.6" @@ -962,9 +1009,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.142" +version = "0.2.171" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a987beff54b60ffa6d51982e1aa1146bc42f19bd26be28b0586f252fccf5317" +checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" [[package]] name = "libsecp256k1" @@ -1059,7 +1106,8 @@ dependencies = [ "crossbeam-channel 0.5.8", "dirs", "ethereum-types", - "itertools", + "ip_country", + "itertools 0.10.5", "lazy_static", "log 0.4.17", "nix", @@ -1067,6 +1115,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", + "test_utilities", "time 0.3.20", "tiny-hderive", "toml", @@ -1085,6 +1134,19 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" +[[package]] +name = "maxminddb" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a197e44322788858682406c74b0b59bf8d9b4954fe1f224d9a25147f1880bba" +dependencies = [ + "ipnetwork", + "log 0.4.17", + "memchr", + "serde", + "thiserror 2.0.12", +] + [[package]] name = "maybe-uninit" version = "2.0.0" @@ -1379,9 +1441,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.56" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" +checksum = "a31971752e70b8b2686d7e46ec17fb38dad4051d94024c88df49b667caea9c84" dependencies = [ "unicode-ident", ] @@ -1403,11 +1465,11 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.26" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" dependencies = [ - "proc-macro2 1.0.56", + "proc-macro2 1.0.94", ] [[package]] @@ -1648,7 +1710,7 @@ checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" dependencies = [ "getrandom 0.2.9", "redox_syscall 0.2.16", - "thiserror", + "thiserror 1.0.40", ] [[package]] @@ -1758,6 +1820,9 @@ name = "serde" version = "1.0.160" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c" +dependencies = [ + "serde_derive", +] [[package]] name = "serde_derive" @@ -1765,9 +1830,9 @@ version = "1.0.160" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "291a097c63d8497e00160b166a967a4a79c64f3facdd01cbd7502231688d77df" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.26", - "syn 2.0.15", + "proc-macro2 1.0.94", + "quote 1.0.40", + "syn 2.0.100", ] [[package]] @@ -1890,19 +1955,19 @@ version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.26", + "proc-macro2 1.0.94", + "quote 1.0.40", "unicode-ident", ] [[package]] name = "syn" -version = "2.0.15" +version = "2.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822" +checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.26", + "proc-macro2 1.0.94", + "quote 1.0.40", "unicode-ident", ] @@ -1912,8 +1977,8 @@ version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.26", + "proc-macro2 1.0.94", + "quote 1.0.40", "syn 1.0.109", "unicode-xid 0.2.4", ] @@ -1927,6 +1992,10 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "test_utilities" +version = "0.1.0" + [[package]] name = "textwrap" version = "0.11.0" @@ -1942,7 +2011,16 @@ version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.40", +] + +[[package]] +name = "thiserror" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +dependencies = [ + "thiserror-impl 2.0.12", ] [[package]] @@ -1951,9 +2029,20 @@ version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.26", - "syn 2.0.15", + "proc-macro2 1.0.94", + "quote 1.0.40", + "syn 2.0.100", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" +dependencies = [ + "proc-macro2 1.0.94", + "quote 1.0.40", + "syn 2.0.100", ] [[package]] @@ -2464,8 +2553,8 @@ dependencies = [ "bumpalo", "log 0.4.17", "once_cell", - "proc-macro2 1.0.56", - "quote 1.0.26", + "proc-macro2 1.0.94", + "quote 1.0.40", "syn 1.0.109", "wasm-bindgen-shared", ] @@ -2476,7 +2565,7 @@ version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" dependencies = [ - "quote 1.0.26", + "quote 1.0.40", "wasm-bindgen-macro-support", ] @@ -2486,8 +2575,8 @@ version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.26", + "proc-macro2 1.0.94", + "quote 1.0.40", "syn 1.0.109", "wasm-bindgen-backend", "wasm-bindgen-shared", diff --git a/ci/all.sh b/ci/all.sh index 91d6b4c8d..6591cd392 100755 --- a/ci/all.sh +++ b/ci/all.sh @@ -8,7 +8,7 @@ ci/format.sh export RUST_BACKTRACE=1 # Remove these two lines to slow down the build -which sccache || cargo install sccache || echo "Skipping sccache installation" # Should do significant work only once +which sccache || cargo install --version 0.4.1 sccache || echo "Skipping sccache installation" # Should do significant work only once #export CARGO_TARGET_DIR="$CI_DIR/../cargo-cache" export SCCACHE_DIR="$HOME/.cargo/sccache" #export RUSTC_WRAPPER="$HOME/.cargo/bin/sccache" @@ -45,4 +45,10 @@ cd "$CI_DIR/../automap" ci/all.sh "$PARENT_DIR" echo "*** AUTOMAP TAIL ***" echo "*********************************************************************************************************" +echo "*********************************************************************************************************" +echo "*** IP COUNTRY HEAD ***" +cd "$CI_DIR/../ip_country" +ci/all.sh "$PARENT_DIR" +echo "*** IP COUNTRY TAIL ***" +echo "*********************************************************************************************************" diff --git a/ci/format.sh b/ci/format.sh index 75747e812..59fa9b34f 100755 --- a/ci/format.sh +++ b/ci/format.sh @@ -25,5 +25,6 @@ format "$CI_DIR"/../dns_utility format "$CI_DIR"/../masq format "$CI_DIR"/../multinode_integration_tests format "$CI_DIR"/../port_exposer +format "$CI_DIR"/../ip_country exit $final_exit_code diff --git a/dns_utility/Cargo.lock b/dns_utility/Cargo.lock index 59f5a3bda..1bba0fb0c 100644 --- a/dns_utility/Cargo.lock +++ b/dns_utility/Cargo.lock @@ -276,8 +276,8 @@ version = "0.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d897f47bf7270cf70d370f8f98c1abb6d2d4cf60a6845d30e05bfb90c6568650" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.26", + "proc-macro2 1.0.94", + "quote 1.0.40", "unicode-xid 0.2.4", ] @@ -399,6 +399,27 @@ dependencies = [ "subtle 1.0.0", ] +[[package]] +name = "csv" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" +dependencies = [ + "memchr", +] + [[package]] name = "digest" version = "0.8.1" @@ -499,8 +520,8 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.26", + "proc-macro2 1.0.94", + "quote 1.0.40", "syn 1.0.109", "synstructure", ] @@ -716,6 +737,17 @@ dependencies = [ "libc", ] +[[package]] +name = "ip_country" +version = "0.1.0" +dependencies = [ + "csv", + "ipnetwork", + "itertools 0.13.0", + "lazy_static", + "maxminddb", +] + [[package]] name = "ipconfig" version = "0.1.9" @@ -741,6 +773,12 @@ dependencies = [ "winreg 0.6.2", ] +[[package]] +name = "ipnetwork" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf370abdafd54d13e54a620e8c3e1145f28e46cc9d704bc6d94414559df41763" + [[package]] name = "itertools" version = "0.10.5" @@ -750,6 +788,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.6" @@ -780,9 +827,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.147" +version = "0.2.171" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" [[package]] name = "libsecp256k1" @@ -862,7 +909,8 @@ dependencies = [ "crossbeam-channel 0.5.8", "dirs", "ethereum-types", - "itertools", + "ip_country", + "itertools 0.10.5", "lazy_static", "log 0.4.17", "nix", @@ -870,6 +918,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", + "test_utilities", "time 0.3.20", "tiny-hderive", "toml", @@ -888,6 +937,19 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" +[[package]] +name = "maxminddb" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a197e44322788858682406c74b0b59bf8d9b4954fe1f224d9a25147f1880bba" +dependencies = [ + "ipnetwork", + "log 0.4.17", + "memchr", + "serde", + "thiserror 2.0.12", +] + [[package]] name = "maybe-uninit" version = "2.0.0" @@ -1139,9 +1201,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.56" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" +checksum = "a31971752e70b8b2686d7e46ec17fb38dad4051d94024c88df49b667caea9c84" dependencies = [ "unicode-ident", ] @@ -1163,11 +1225,11 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.26" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" dependencies = [ - "proc-macro2 1.0.56", + "proc-macro2 1.0.94", ] [[package]] @@ -1368,7 +1430,7 @@ checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" dependencies = [ "getrandom 0.2.10", "redox_syscall 0.2.16", - "thiserror", + "thiserror 1.0.40", ] [[package]] @@ -1472,6 +1534,9 @@ name = "serde" version = "1.0.160" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c" +dependencies = [ + "serde_derive", +] [[package]] name = "serde_derive" @@ -1479,9 +1544,9 @@ version = "1.0.160" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "291a097c63d8497e00160b166a967a4a79c64f3facdd01cbd7502231688d77df" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.26", - "syn 2.0.15", + "proc-macro2 1.0.94", + "quote 1.0.40", + "syn 2.0.100", ] [[package]] @@ -1604,19 +1669,19 @@ version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.26", + "proc-macro2 1.0.94", + "quote 1.0.40", "unicode-ident", ] [[package]] name = "syn" -version = "2.0.15" +version = "2.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822" +checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.26", + "proc-macro2 1.0.94", + "quote 1.0.40", "unicode-ident", ] @@ -1626,8 +1691,8 @@ version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.26", + "proc-macro2 1.0.94", + "quote 1.0.40", "syn 1.0.109", "unicode-xid 0.2.4", ] @@ -1653,6 +1718,10 @@ dependencies = [ "libc", ] +[[package]] +name = "test_utilities" +version = "0.1.0" + [[package]] name = "textwrap" version = "0.11.0" @@ -1668,7 +1737,16 @@ version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.40", +] + +[[package]] +name = "thiserror" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +dependencies = [ + "thiserror-impl 2.0.12", ] [[package]] @@ -1677,9 +1755,20 @@ version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.26", - "syn 2.0.15", + "proc-macro2 1.0.94", + "quote 1.0.40", + "syn 2.0.100", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" +dependencies = [ + "proc-macro2 1.0.94", + "quote 1.0.40", + "syn 2.0.100", ] [[package]] diff --git a/ip_country/Cargo.lock b/ip_country/Cargo.lock new file mode 100644 index 000000000..89f8e2b11 --- /dev/null +++ b/ip_country/Cargo.lock @@ -0,0 +1,179 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "csv" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" +dependencies = [ + "memchr", +] + +[[package]] +name = "either" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" + +[[package]] +name = "ip_country" +version = "0.1.0" +dependencies = [ + "csv", + "ipnetwork", + "itertools", + "lazy_static", + "maxminddb", + "test_utilities", +] + +[[package]] +name = "ipnetwork" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf370abdafd54d13e54a620e8c3e1145f28e46cc9d704bc6d94414559df41763" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "log" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" + +[[package]] +name = "maxminddb" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a197e44322788858682406c74b0b59bf8d9b4954fe1f224d9a25147f1880bba" +dependencies = [ + "ipnetwork", + "log", + "memchr", + "serde", + "thiserror", +] + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "proc-macro2" +version = "1.0.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31971752e70b8b2686d7e46ec17fb38dad4051d94024c88df49b667caea9c84" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + +[[package]] +name = "serde" +version = "1.0.205" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e33aedb1a7135da52b7c21791455563facbbcc43d0f0f66165b42c21b3dfb150" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.205" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "692d6f5ac90220161d6774db30c662202721e64aed9058d2c394f451261420c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "syn" +version = "2.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "test_utilities" +version = "0.1.0" + +[[package]] +name = "thiserror" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" diff --git a/ip_country/Cargo.toml b/ip_country/Cargo.toml new file mode 100644 index 000000000..5b647ca6f --- /dev/null +++ b/ip_country/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "ip_country" +version = "0.1.0" +edition = "2021" +license = "GPL-3.0-only" +authors = ["Dan Wiebe ", "MASQ"] +description = "Handle embedding IP-address-to-country data in MASQ Node" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +csv = "1.3.0" +ipnetwork = "0.21.0" +itertools = "0.13.0" +lazy_static = "1.4.0" +maxminddb = "0.26.0" + +[dev-dependencies] +test_utilities = { path = "../test_utilities"} + +[[bin]] +name = "ip_country" +path = "src/main.rs" + +[lib] +name = "ip_country_lib" +path = "src/lib.rs" diff --git a/ip_country/ci/all.sh b/ip_country/ci/all.sh new file mode 100755 index 000000000..af62832cb --- /dev/null +++ b/ip_country/ci/all.sh @@ -0,0 +1,8 @@ +#!/bin/bash -xev +# Copyright (c) 2024, MASQ (https://masq.ai) and/or its affiliates. All rights reserved. +CI_DIR="$( cd "$( dirname "$0" )" && pwd )" + +pushd "$CI_DIR/.." +ci/lint.sh +ci/unit_tests.sh +popd diff --git a/ip_country/ci/build.sh b/ip_country/ci/build.sh new file mode 100755 index 000000000..f9fa1fa7c --- /dev/null +++ b/ip_country/ci/build.sh @@ -0,0 +1,7 @@ +#!/bin/bash -xev +# Copyright (c) 2024, MASQ (https://masq.ai) and/or its affiliates. All rights reserved. +CI_DIR="$( cd "$( dirname "$0" )" && pwd )" + +pushd "$CI_DIR/.." +cargo build --release --verbose +popd \ No newline at end of file diff --git a/ip_country/ci/download_dbip.sh b/ip_country/ci/download_dbip.sh new file mode 100644 index 000000000..a99613196 --- /dev/null +++ b/ip_country/ci/download_dbip.sh @@ -0,0 +1,121 @@ +#!/bin/bash + +set -e + +# Configuration +TARGET_DIR="src" +DB_FILENAME="dbip.mmdb" +DOWNLOAD_PAGE="https://db-ip.com/db/download/ip-to-country-lite" +TEMP_DIR=$(mktemp -d) + +# Parse command line arguments +while [[ $# -gt 0 ]]; do + case $1 in + --output-dir) + TARGET_DIR="$2" + shift 2 + ;; + --filename) + DB_FILENAME="$2" + shift 2 + ;; + --force) + FORCE_DOWNLOAD=1 + shift + ;; + --help) + echo "Usage: $0 [OPTIONS]" + echo "Options:" + echo " --output-dir DIR Directory to store the database (default: src)" + echo " --filename NAME Name of the output file (default: dbip.mmdb)" + echo " --force Force download even if file exists" + echo " --help Show this help message" + exit 0 + ;; + *) + echo "Unknown option: $1" + echo "Use --help for usage information" + exit 1 + ;; + esac +done + +# Ensure cleanup on script exit +cleanup() { + rm -rf "$TEMP_DIR" +} +trap cleanup EXIT + +# Check if curl is available +if ! command -v curl >/dev/null 2>&1; then + echo "Error: curl is not installed" + exit 1 +fi + +# Check if target file already exists +TARGET_PATH="$TARGET_DIR/$DB_FILENAME" +if [ -f "$TARGET_PATH" ] && [ -z "$FORCE_DOWNLOAD" ]; then + echo "Database file already exists at $TARGET_PATH" + echo "Use --force to download anyway" + exit 0 +fi + +echo "Fetching download page..." +# Fetch the download page and extract the latest download link +DOWNLOAD_URL=$(curl -s "$DOWNLOAD_PAGE" | grep -o 'https://download\.db-ip\.com/free/dbip-country-lite-[0-9][0-9][0-9][0-9]-[0-9][0-9]\.mmdb\.gz' | head -1) + +# Check if we found a URL +if [ -z "$DOWNLOAD_URL" ]; then + echo "Error: Failed to find the download link on the page" + exit 1 +fi + +# Extract date from URL for version checking +DB_DATE=$(echo "$DOWNLOAD_URL" | grep -o '[0-9][0-9][0-9][0-9]-[0-9][0-9]') +echo "Found database version: $DB_DATE" + +# Create target directory if it doesn't exist +mkdir -p "$TARGET_DIR" + +# Download the latest dbip IP-to-country data with progress bar +echo "Downloading database from $DOWNLOAD_URL..." +if ! curl -L --progress-bar -o "$TEMP_DIR/dbip.mmdb.gz" "$DOWNLOAD_URL"; then + echo "Error: Download failed" + exit 1 +fi + +# Verify downloaded file +if [ ! -s "$TEMP_DIR/dbip.mmdb.gz" ]; then + echo "Error: Downloaded file is empty" + exit 1 +fi + +# Extract the data +echo "Extracting database..." +if ! gunzip -f "$TEMP_DIR/dbip.mmdb.gz"; then + echo "Error: Failed to extract the database" + exit 1 +fi + +# Verify the extracted file +if [ ! -s "$TEMP_DIR/dbip.mmdb" ]; then + echo "Error: Extracted database is empty" + exit 1 +fi + +# Move the extracted file to the target directory +echo "Moving database to $TARGET_PATH..." +if ! mv "$TEMP_DIR/dbip.mmdb" "$TARGET_PATH"; then + echo "Error: Failed to move database to target location" + # Restore backup if it exists + if [ -f "${TARGET_PATH}.bak" ]; then + mv "${TARGET_PATH}.bak" "$TARGET_PATH" + fi + exit 1 +fi + +# Remove backup if move was successful +rm -f "${TARGET_PATH}.bak" + +echo "Successfully downloaded and installed database version $DB_DATE to $TARGET_PATH" +echo "You can now run 'cargo run --bin generate_dbip' to update the Rust code" diff --git a/ip_country/ci/license.sh b/ip_country/ci/license.sh new file mode 100755 index 000000000..28dfb6291 --- /dev/null +++ b/ip_country/ci/license.sh @@ -0,0 +1,25 @@ +#!/bin/bash -e +# Copyright (c) 2024, MASQ (https://masq.ai) and/or its affiliates. All rights reserved. +CI_DIR="$( cd "$( dirname "$0" )" && pwd )" + +pushd "$CI_DIR/.." +echo "Checking licenses referenced in Cargo.toml dependencies..." +shopt -s nocasematch +license_errors=$(date +"./%Y-%m-%d-%H%M-license-errors.tmp") +while read -r line; do + if [[ ! $line =~ warning..ianal.* ]]; then + if [[ $line =~ error.* ]]; then + echo $line >> $license_errors + elif [[ $line =~ warning.* ]]; then + echo $line >> $license_errors + fi + fi +done < <(cargo lichking check --all 2>&1) + +if [[ -s $license_errors ]]; then + cat $license_errors + rm $license_errors + exit 1 +fi +echo "License check successful!" +popd diff --git a/ip_country/ci/lint.sh b/ip_country/ci/lint.sh new file mode 100755 index 000000000..acd4c8056 --- /dev/null +++ b/ip_country/ci/lint.sh @@ -0,0 +1,7 @@ +#!/bin/bash -xev +# Copyright (c) 2024, MASQ (https://masq.ai) and/or its affiliates. All rights reserved. +CI_DIR="$( cd "$( dirname "$0" )" && pwd )" + +pushd "$CI_DIR/.." +cargo clippy -- -D warnings +popd diff --git a/ip_country/ci/unit_tests.sh b/ip_country/ci/unit_tests.sh new file mode 100755 index 000000000..424b1dda2 --- /dev/null +++ b/ip_country/ci/unit_tests.sh @@ -0,0 +1,10 @@ +#!/bin/bash -xev +# Copyright (c) 2024, MASQ (https://masq.ai) and/or its affiliates. All rights reserved. +CI_DIR="$( cd "$( dirname "$0" )" && pwd )" + +export RUST_BACKTRACE=full +export RUSTFLAGS="-D warnings" +pushd "$CI_DIR/.." + +cargo test --release -- --nocapture --skip _integration +popd diff --git a/ip_country/data/corrupted.mmdb b/ip_country/data/corrupted.mmdb new file mode 100644 index 000000000..e15f3e67e Binary files /dev/null and b/ip_country/data/corrupted.mmdb differ diff --git a/ip_country/data/country-scratch-out.mmdb b/ip_country/data/country-scratch-out.mmdb new file mode 100644 index 000000000..e7c4f7acb Binary files /dev/null and b/ip_country/data/country-scratch-out.mmdb differ diff --git a/ip_country/data/improperly-formatted.mmdb b/ip_country/data/improperly-formatted.mmdb new file mode 100644 index 000000000..0ce87604c --- /dev/null +++ b/ip_country/data/improperly-formatted.mmdb @@ -0,0 +1,6 @@ +Four score and seven years ago our fathers brought forth on this continent, a new nation, conceived in Liberty, and dedicated to the proposition that all men are created equal. + +Now we are engaged in a great civil war, testing whether that nation, or any nation so conceived and so dedicated, can long endure. We are met on a great battle-field of that war. We have come to dedicate a portion of that field, as a final resting place for those who here gave their lives that that nation might live. It is altogether fitting and proper that we should do this. + +But, in a larger sense, we can not dedicate -- we can not consecrate -- we can not hallow -- this ground. The brave men, living and dead, who struggled here, have consecrated it, far above our poor power to add or detract. The world will little note, nor long remember what we say here, but it can never forget what they did here. It is for us the living, rather, to be dedicated here to the unfinished work which they who fought here have thus far so nobly advanced. It is rather for us to be here dedicated to the great task remaining before us -- that from these honored dead we take increased devotion to that cause for which they gave the last full measure of devotion -- that we here highly resolve that these dead shall not have died in vain -- that this nation, under God, shall have a new birth of freedom -- and that government of the people, by the people, for the people, shall not perish from the earth. + diff --git a/ip_country/src/bit_queue.rs b/ip_country/src/bit_queue.rs new file mode 100644 index 000000000..95a96d3c1 --- /dev/null +++ b/ip_country/src/bit_queue.rs @@ -0,0 +1,411 @@ +// Copyright (c) 2024, MASQ (https://masq.ai) and/or its affiliates. All rights reserved. + +use std::collections::VecDeque; + +#[derive(Debug)] +pub struct BitQueue { + back_blank_bit_count: usize, // number of high-order bits in the back byte of the queue that are unused + byte_queue: VecDeque, + front_blank_bit_count: usize, // number of low-order bits in the front byte of the queue that are unused +} + +impl Default for BitQueue { + fn default() -> Self { + Self::new() + } +} + +impl BitQueue { + pub fn new() -> Self { + let byte_queue = VecDeque::from(vec![0, 0]); + Self { + back_blank_bit_count: 8, + byte_queue, + front_blank_bit_count: 8, + } + } + + pub fn len(&self) -> usize { + (self.byte_queue.len() * 8) - self.back_blank_bit_count - self.front_blank_bit_count + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + #[allow(unused_assignments)] + pub fn add_bits(&mut self, mut bit_data: u64, mut bit_count: usize) { + if bit_count > 64 { + panic!( + "You can only add bits up to 64 at a time, not {}", + bit_count + ) + } + let initial_bits_added = self.add_some_back_bits(bit_data, bit_count); + bit_data >>= initial_bits_added; + bit_count -= initial_bits_added; + let byte_bits_added = self.add_back_bytes(bit_data, bit_count); + bit_data >>= byte_bits_added; + bit_count -= byte_bits_added; + let final_bits_added = self.add_some_back_bits(bit_data, bit_count); + bit_data >>= final_bits_added; + bit_count -= final_bits_added; + if bit_count != 0 { + panic!("Didn't add all the bits: {} left", bit_count); + } + } + + #[allow(unused_assignments)] + pub fn take_bits(&mut self, mut bit_count: usize) -> Option { + let original_bit_count = bit_count; + if bit_count > 64 { + panic!( + "You can only take bits up to 64 at a time, not {}", + bit_count + ) + } + if bit_count > self.len() { + return None; + } + let mut bit_data = 0u64; + let mut bit_position = 0usize; + + let (initial_bit_data, initial_bit_count) = self.take_some_front_bits(bit_count); + bit_data |= initial_bit_data << bit_position; + bit_position += initial_bit_count; + bit_count -= initial_bit_count; + + let (byte_bit_data, byte_bit_count) = self.take_front_bytes(bit_count); + bit_data |= byte_bit_data << bit_position; + bit_position += byte_bit_count; + bit_count -= byte_bit_count; + + let (final_front_bit_data, final_front_bit_count) = self.take_some_front_bits(bit_count); + if final_front_bit_count > 0 { + bit_data |= final_front_bit_data << bit_position; + bit_position += final_front_bit_count; + } + bit_count -= final_front_bit_count; + + let (final_back_bit_data, final_back_bit_count) = self.take_some_back_bits(bit_count); + if final_back_bit_count > 0 { + bit_data |= final_back_bit_data << bit_position; + bit_position += final_back_bit_count; + } + bit_count -= final_back_bit_count; + if bit_position != original_bit_count { + panic!( + "Wanted {} bits, but got {} instead", + original_bit_count, bit_position + ); + } + Some(bit_data) + } + + fn back_full_bit_count(&self) -> usize { + 8 - self.back_blank_bit_count + } + + fn front_full_bit_count(&self) -> usize { + 8 - self.front_blank_bit_count + } + + fn low_order_ones(count: usize) -> u64 { + !(u64::MAX << count) + } + + fn add_some_back_bits(&mut self, bit_data: u64, bit_count: usize) -> usize { + if self.back_blank_bit_count == 0 { + self.byte_queue.push_back(0); + self.back_blank_bit_count = 8; + } + let bits_to_add = bit_count.min(self.back_blank_bit_count); + let back_full_bit_count = self.back_full_bit_count(); + let back_ref = self + .byte_queue + .back_mut() + .expect("There should be a back byte"); + *back_ref |= (bit_data << back_full_bit_count) as u8; + self.back_blank_bit_count -= bits_to_add; + if self.back_blank_bit_count == 0 { + self.byte_queue.push_back(0); + self.back_blank_bit_count = 8; + } + bits_to_add + } + + fn add_back_bytes(&mut self, mut bit_data: u64, mut bit_count: usize) -> usize { + let original_bit_count = bit_count; + if bit_count > 64 { + panic!( + "add_back_bytes() can add a maximum of 64 bits per call, not {}", + bit_count + ) + } + if bit_count < 8 { + return 0; + } + if self.back_blank_bit_count == 8 { + let _ = self.byte_queue.pop_back(); + self.back_blank_bit_count = 0; + } + if self.back_blank_bit_count > 0 { + panic!( + "add_back_bytes() only works when there are no back blank bits, not {}", + self.back_blank_bit_count + ) + } + while bit_count >= 8 { + let next_byte = (bit_data & Self::low_order_ones(8)) as u8; + self.byte_queue.push_back(next_byte); + bit_data >>= 8; + bit_count -= 8; + } + original_bit_count - bit_count + } + + fn take_some_front_bits(&mut self, bit_count: usize) -> (u64, usize) { + if (self.front_full_bit_count() == 0) && (self.byte_queue.len() > 2) { + let _ = self.byte_queue.pop_front(); + self.front_blank_bit_count = 0; + } + let bits_to_take = bit_count.min(self.front_full_bit_count()); + if bits_to_take == 0 { + return (0, 0); + } + let front_ref = self + .byte_queue + .front_mut() + .expect("There should be a front byte"); + let bit_data = *front_ref & (Self::low_order_ones(bits_to_take) as u8); + *front_ref = if bits_to_take < 8 { + *front_ref >> bits_to_take + } else { + 0 + }; + self.front_blank_bit_count += bits_to_take; + if (self.front_blank_bit_count == 8) && (self.byte_queue.len() > 2) { + let _ = self.byte_queue.pop_front(); + self.front_blank_bit_count = 0; + } + (bit_data as u64, bits_to_take) + } + + fn take_front_bytes(&mut self, mut bit_count: usize) -> (u64, usize) { + let original_bit_count = bit_count; + if bit_count > 64 { + panic!( + "take_front_bytes() can take a maximum of 64 bits per call, not {}", + bit_count + ) + } + if bit_count < 8 { + return (0, 0); + } + if self.front_blank_bit_count == 8 { + let _ = self.byte_queue.pop_front(); + self.front_blank_bit_count = 0; + } + if self.front_blank_bit_count > 0 { + panic!( + "take_front_bytes() only works when there are no front blank bits, not {}", + self.front_blank_bit_count + ) + } + let mut bit_data = 0u64; + while bit_count >= 8 { + let byte = self + .byte_queue + .pop_front() + .expect("Demanded too many bytes") as u64; + bit_data |= byte << (original_bit_count - bit_count); + bit_count -= 8; + } + if self.byte_queue.len() < 2 { + self.byte_queue.push_front(0); + self.front_blank_bit_count = 8; + } + (bit_data, original_bit_count - bit_count) + } + + fn take_some_back_bits(&mut self, bit_count: usize) -> (u64, usize) { + let bits_to_take = bit_count.min(self.back_full_bit_count()); + let remaining_bits = self.back_full_bit_count() - bits_to_take; + let mask = Self::low_order_ones(bits_to_take); + let back_ref = self + .byte_queue + .back_mut() + .expect("There should be a back byte"); + let bit_data = if remaining_bits < 8 { + *back_ref & (mask as u8) + } else { + 0 + }; + *back_ref >>= bits_to_take; + self.back_blank_bit_count += bits_to_take; + if (self.back_blank_bit_count == 8) && (self.byte_queue.len() > 2) { + let _ = self.byte_queue.pop_back(); + self.back_blank_bit_count = 0; + } + (bit_data as u64, bits_to_take) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reading_without_writing_produces_none() { + let mut subject = BitQueue::new(); + + let result = subject.take_bits(1); + + assert_eq!(result, None); + } + + #[test] + #[should_panic(expected = "You can only add bits up to 64 at a time, not 65")] + fn adding_more_than_64_bits_causes_panic() { + let mut subject = BitQueue::new(); + + subject.add_bits(0, 65); + } + + #[test] + #[should_panic(expected = "You can only take bits up to 64 at a time, not 65")] + fn taking_more_than_64_bits_causes_panic() { + let mut subject = BitQueue::new(); + + let _ = subject.take_bits(65); + } + + #[test] + fn queues_and_dequeues_one_bit() { + let mut subject = BitQueue::new(); + + subject.add_bits(1, 1); + let one_bit = subject.take_bits(1); + subject.add_bits(0, 1); + let zero_bit = subject.take_bits(1); + + assert_eq!(one_bit, Some(1)); + assert_eq!(zero_bit, Some(0)); + } + + #[test] + fn queues_and_dequeues_seven_bits() { + let mut subject = BitQueue::new(); + + subject.add_bits(0b1101101, 7); + let seven_bits = subject.take_bits(7).unwrap(); + + assert_bit_field(seven_bits, 0b1101101); + assert_eq!(subject.len(), 0); + } + + #[test] + fn queues_and_dequeues_nine_bits() { + let mut subject = BitQueue::new(); + + subject.add_bits(0b110110111, 9); + let nine_bits = subject.take_bits(9).unwrap(); + + assert_bit_field(nine_bits, 0b110110111); + assert_eq!(subject.len(), 0); + } + + #[test] + fn nine_and_seven_then_nine_and_seven() { + let mut subject = BitQueue::new(); + let sixteen_bits = 0b1000001100000001u64; + + subject.add_bits(sixteen_bits & 0x1FF, 9); + assert_eq!(subject.len(), 9); + subject.add_bits(sixteen_bits >> 9, 7); + assert_eq!(subject.len(), 16); + let nine_bits = subject.take_bits(9).unwrap(); + assert_eq!(subject.len(), 7); + let seven_bits = subject.take_bits(7).unwrap(); + assert_eq!(subject.len(), 0); + + assert_bit_field(nine_bits, sixteen_bits & 0x1FF); + assert_bit_field(seven_bits, sixteen_bits >> 9); + } + + #[test] + fn nine_and_seven_then_seven_and_nine() { + let mut subject = BitQueue::new(); + let sixteen_bits = 0b1000000110000001u64; + + subject.add_bits(sixteen_bits & 0x1FF, 9); + assert_eq!(subject.len(), 9); + subject.add_bits(sixteen_bits >> 9, 7); + assert_eq!(subject.len(), 16); + let seven_bits = subject.take_bits(7).unwrap(); + assert_eq!(subject.len(), 9); + let nine_bits = subject.take_bits(9).unwrap(); + assert_eq!(subject.len(), 0); + + assert_bit_field(seven_bits, sixteen_bits & 0x7F); + assert_bit_field(nine_bits, sixteen_bits >> 7); + } + + #[test] + fn seven_and_nine_then_nine_and_seven() { + let mut subject = BitQueue::new(); + + subject.add_bits(0b0101100, 7); + assert_eq!(subject.len(), 7); + subject.add_bits(0b110011101, 9); + assert_eq!(subject.len(), 16); + let nine_bits = subject.take_bits(9).unwrap(); + assert_eq!(subject.len(), 7); + let seven_bits = subject.take_bits(7).unwrap(); + assert_eq!(subject.len(), 0); + + assert_bit_field(nine_bits, 0b010101100); + assert_bit_field(seven_bits, 0b1100111); + assert_eq!(subject.len(), 0); + } + + #[test] + fn queues_and_dequeues_32_bits() { + let value: u64 = 0xDEADBEEF; + let mut subject = BitQueue::new(); + + subject.add_bits(value, 32); + let result = subject.take_bits(32).unwrap(); + + assert_eq!( + result, value, + "Should have been {:08X}, but was {:08X}", + value, result + ); + } + + #[test] + fn can_queue_bits_properly() { + let mut subject = BitQueue::new(); + + subject.add_bits(0b11011, 5); + subject.add_bits(0b00111001110011100, 17); + subject.add_bits(0b1, 1); + + let first_chunk = subject.take_bits(10).unwrap(); + let second_chunk = subject.take_bits(5).unwrap(); + let third_chunk = subject.take_bits(5).unwrap(); + let fourth_chunk = subject.take_bits(3).unwrap(); + let should_be_none = subject.take_bits(1); + + assert_bit_field(first_chunk, 0b1110011011); + assert_bit_field(second_chunk, 0b11100); + assert_bit_field(third_chunk, 0b11100); + assert_bit_field(fourth_chunk, 0b100); + assert_eq!(should_be_none, None); + } + + fn assert_bit_field(actual: u64, expected: u64) { + assert_eq!(actual, expected, "Got {:b}, wanted {:b}", actual, expected) + } +} diff --git a/ip_country/src/countries.rs b/ip_country/src/countries.rs new file mode 100644 index 000000000..0f628bc81 --- /dev/null +++ b/ip_country/src/countries.rs @@ -0,0 +1,291 @@ +// Copyright (c) 2024, MASQ (https://masq.ai) and/or its affiliates. All rights reserved. + +use crate::country_block_stream::Country; +use std::collections::HashMap; + +#[derive(Debug, PartialEq, Eq, Clone)] +pub struct Countries { + countries: Vec, + index_by_iso3166: HashMap, +} + +impl Countries { + pub fn old_new(countries: Vec) -> Self { + let index_by_iso3166 = countries + .iter() + .map(|country| (country.iso3166.clone(), country.index)) + .collect::>(); + Self { + countries, + index_by_iso3166, + } + } + + pub fn new(mut country_pairs: Vec<(String, String)>) -> Self { + // Must sort these by iso3166, but we need to keep the sentinel coded as "ZZ" at the front + // --or add one at the front, if there isn't already one. (We assume there isn't already + // more than one.) + let sentinel_info_opt = country_pairs + .iter() + .enumerate() + .find(|(_, (iso3166, _))| iso3166 == "ZZ") + .map(|(index, (_, name))| (index, name.to_string())); + if let Some((index, _)) = sentinel_info_opt { + country_pairs.remove(index); + } + country_pairs.sort_by(|a, b| a.0.cmp(&b.0)); + let mut countries = country_pairs + .iter() + .enumerate() + .map(|(index, (iso3166, name))| Country::new(index + 1, iso3166, name)) + .collect::>(); + let sentinel = Country::new(0, "ZZ", "Sentinel"); + countries.insert(0, sentinel); + Self::old_new(countries) + } + + pub fn country_from_code(&self, iso3166: &str) -> Result<&Country, String> { + let index = match self.index_by_iso3166.get(&iso3166.to_ascii_uppercase()) { + None => return Err(format!("'{}' is not a valid ISO3166 country code", iso3166)), + Some(index) => *index, + }; + let country = self.country_from_index(index).unwrap_or_else(|_| { + panic!( + "Data error: ISO3166 {} maps to index {}, but there is no such Country", + iso3166, index + ) + }); + Ok(country) + } + + pub fn country_from_index(&self, index: usize) -> Result<&Country, String> { + match self.countries.get(index) { + None => Err(format!( + "There are only {} Countries; no Country is at index {}", + self.countries.len(), + index + )), + Some(country) => Ok(country), + } + } + + pub fn iter(&self) -> impl Iterator { + self.countries.iter() + } + + #[allow(clippy::len_without_is_empty)] // A Countries object is never empty: always has Sentinel + pub fn len(&self) -> usize { + self.countries.len() + } +} + +// impl TryFrom<&str> for Country { +// type Error = String; +// +// fn try_from(iso3166: &str) -> Result { +// let index = match INDEX_BY_ISO3166.get(&iso3166.to_ascii_uppercase()) { +// None => return Err(format!("'{}' is not a valid ISO3166 country code", iso3166)), +// Some(index) => *index, +// }; +// let country = Country::try_from(index).unwrap_or_else(|_| { +// panic!( +// "Data error: ISO3166 {} maps to index {}, but there is no such Country", +// iso3166, index +// ) +// }); +// Ok(country) +// } +// } +// +// impl From for Country { +// fn from(index: usize) -> Self { +// match COUNTRIES.get(index) { +// None => panic!( +// "There are only {} Countries; no Country is at index {}", +// COUNTRIES.len(), +// index +// ), +// Some(country) => country.clone(), +// } +// } +// } + +#[cfg(test)] +mod tests { + use crate::countries::Countries; + use crate::country_block_stream::Country; + use crate::dbip_country::COUNTRIES; + use itertools::Itertools; + + #[test] + fn countries_without_a_sentinel_grow_one() { + let country_pairs = vec![ + ("AD", "Andorra"), + ("AO", "Angola"), + ("AS", "American Samoa"), + ] + .into_iter() + .map(|(code, name)| (code.to_string(), name.to_string())) + .collect::>(); + + let subject = Countries::new(country_pairs); + + assert_eq!(subject.len(), 4); + assert_eq!( + subject.country_from_code("ZZ").unwrap(), + &Country::new(0, "ZZ", "Sentinel") + ); + } + + #[test] + fn countries_with_a_misplaced_sentinel_relocate_it() { + let country_pairs = vec![ + ("AD", "Andorra"), + ("AO", "Angola"), + ("ZZ", "Sentinel"), + ("AS", "American Samoa"), + ] + .into_iter() + .map(|(code, name)| (code.to_string(), name.to_string())) + .collect::>(); + + let subject = Countries::new(country_pairs); + + assert_eq!(subject.len(), 4); + assert_eq!( + subject.country_from_code("ZZ").unwrap(), + &Country::new(0, "ZZ", "Sentinel") + ); + } + + #[test] + fn countries_with_a_misnamed_sentinel_rename_it() { + let country_pairs = vec![ + ("AD", "Andorra"), + ("AO", "Angola"), + ("ZZ", "Something Other Than Sentinel, Perhaps 'Undefined'"), + ("AS", "American Samoa"), + ] + .into_iter() + .map(|(code, name)| (code.to_string(), name.to_string())) + .collect::>(); + + let subject = Countries::new(country_pairs); + + assert_eq!(subject.len(), 4); + assert_eq!( + subject.country_from_code("ZZ").unwrap(), + &Country::new(0, "ZZ", "Sentinel") + ); + } + + #[test] + fn sentinel_is_first() { + let sentinel = COUNTRIES.countries.get(0).unwrap(); + + assert_eq!(sentinel.iso3166.as_str(), "ZZ"); + assert_eq!(sentinel.name.as_str(), "Sentinel"); + } + + #[test] + fn countries_are_properly_ordered() { + COUNTRIES + .countries + .iter() + .skip(1) + .tuple_windows() + .for_each(|(a, b)| { + assert!( + a.iso3166 < b.iso3166, + "Country code {} should have come before {}, but was after", + b.iso3166, + a.iso3166 + ) + }); + } + + #[test] + fn countries_are_properly_indexed() { + COUNTRIES + .countries + .iter() + .enumerate() + .for_each(|(index, country)| { + assert_eq!( + country.index, index, + "Index for {} should have been {} but was {}", + country.name, index, country.index + ) + }); + } + + #[test] + fn string_length_check() { + COUNTRIES.countries.iter().for_each(|country| { + assert_eq!(country.iso3166.len(), 2); + assert_eq!( + country.name.len() > 0, + true, + "Blank country name for {} at index {}", + country.iso3166, + country.index + ); + }) + } + + #[test] + fn try_from_str_happy_path() { + for country in COUNTRIES.countries.iter() { + let result = COUNTRIES + .country_from_code(country.iso3166.as_str()) + .unwrap(); + + assert_eq!(result, country); + } + } + + #[test] + fn try_from_str_wrong_case() { + for country in COUNTRIES.countries.iter() { + let result = COUNTRIES + .country_from_code(country.iso3166.to_lowercase().as_str()) + .unwrap(); + + assert_eq!(result, country); + } + } + + #[test] + fn try_from_str_bad_iso3166() { + let result = COUNTRIES.country_from_code("Booga"); + + assert_eq!( + result, + Err("'Booga' is not a valid ISO3166 country code".to_string()) + ); + } + + #[test] + fn from_index_happy_path() { + for country in COUNTRIES.countries.iter() { + let result = COUNTRIES.country_from_index(country.index).unwrap(); + + assert_eq!(result, country); + } + } + + #[test] + fn try_from_index_bad_index() { + let count = COUNTRIES.len(); + + let result = COUNTRIES.country_from_index(4096usize).err().unwrap(); + + assert_eq!( + result, + format!( + "There are only {} Countries; no Country is at index 4096", + count + ) + ); + } +} diff --git a/ip_country/src/country_block_serde.rs b/ip_country/src/country_block_serde.rs new file mode 100644 index 000000000..8a98ca4f2 --- /dev/null +++ b/ip_country/src/country_block_serde.rs @@ -0,0 +1,1430 @@ +// Copyright (c) 2024, MASQ (https://masq.ai) and/or its affiliates. All rights reserved. + +use crate::bit_queue::BitQueue; +use crate::countries::Countries; +use crate::country_block_serde::semi_private_items::{ + DeserializerPrivate, Difference, IPIntoOctets, IPIntoSegments, PlusMinusOneIP, +}; +use crate::country_block_stream::{CountryBlock, IpRange}; +use std::fmt::Debug; +use std::marker::PhantomData; +use std::net::{Ipv4Addr, Ipv6Addr}; +use std::ops::{BitOrAssign, ShlAssign}; + +/* + +Compressed Data Format + +Country IP-address data is stored in compressed format as a literal Vec. In order to +traverse it, it's fed into a BitQueue and then retrieved as a series of variably-sized bit strings. +IPv4 data is stored in one Vec, and IPv6 data is stored in a different one. + +Conceptually, the compressed data format is a sequence of two-element records: + +, +, +, +[...] + +Each block is assumed to end at the address immediately before the one where the next block starts. +If the data contains no block starting at the lowest possible address (0.0.0.0 for IPv4, +0:0:0:0:0:0:0:0 for IPv6), the deserializer will invent one starting at that address, ending just +before the first address specified in the data, and having country index 0. The last block ends at +the maximum possible address: 255.255.255.255 for IPv4, FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF +for IPv6. + +The index of the country in the COUNTRIES list is specified as nine bits. At the time this code was +written, there were 250 countries in ISO3166, so we could have used eight bits; but 250 is close +enough to 256 that we added an extra bit for future-proofing. + +The block-start IP addresses are specified in compressed fashion. Only the parts (octets for IPv4, +segments for IPv6) of the start address that are different from the corresponding segments of the +previous address are stored, like this: + + + +For IPv4, the difference-count-minus-one is stored as two bits, and for IPv6 it's stored as three +bits. Make sure you add 1 to the value before you use it. (The data is stored this way because +there can't be zero changes (that'd imply a zero-length block), but it _is_ possible that every part +of the new start address is different, and that number wouldn't fit into the available bit field.) + +Each difference is stored as two fields: an index and a value, like this: + + + +The index refers to the number of the address part that's different, and the value is the new +address part. For IPv4, the index is two bits long and the value is eight bits long. For IPv6, +the index is three bits long and the value is sixteen bits long. + +Since every start address is composed of differences from the address before it, the very first +start address is compared against 255.255.255.254 for IPv4 and +FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFE for IPv6. + +Examples + +Here are two compressed IPv4 records, with whitespace for clarity: + +11 00 00000001 01 00000010 10 00000011 11 00000100 011101101 +00 00 11111111 011101011 + +This says: +1. There are 4 (3 + 1) differences. Octet 0 is 1; octet 1 is 2; octet 2 is 3; octet 3 is 4. The +country index is 237, which is "US" in the COUNTRIES list. +2. There is 1 (0 + 1) difference. Octet 0 changes to 255. The country index is 235, which is "GB" +in the COUNTRIES list. + +It would deserialize into three CountryBlocks: +CountryBlock { // this block is implied by the fact that the first start address wasn't 0.0.0.0 + pub ip_range: IpRange::V4(Ipv4Addr.from_str("0.0.0.0").unwrap(), Ipv4Addr.from_str("1.2.3.3").unwrap()), + pub country: Country::try_from("ZZ").unwrap(), // generated blocks are always for ZZ +} +CountryBlock { + pub ip_range: IpRange::V4(Ipv4Addr.from_str("1.2.3.4").unwrap(), Ipv4Addr.from_str("255.2.3.3").unwrap()), + pub country: Country::try_from("US").unwrap(), +} +CountryBlock { + pub ip_range: IpRange::V4(Ipv4Addr.from_str("255.2.3.4").unwrap(), Ipv4Addr.from_str("255.255.255.255").unwrap()), + pub country: Country::try_from("GB").unwrap(), +} + +Here are two compressed IPv6 records, with whitespace for clarity: + +111 + 000 0000000000000001 + 001 0000000000000010 + 010 0000000000000011 + 011 0000000000000100 + 100 0000000000000000 + 101 0000000000000000 + 110 0000000000000000 + 111 0000000000000000 + 011101101 +000 + 000 0000000011111111 + 011101011 + +This says: +1. There are 8 (7 + 1) differences. Segment 0 is 1; segment 1 is 2; segment 2 is 3; segment 3 is 4; +and the other four segments are all 0. The country index is 237, which is "US" in the COUNTRIES +list. +2. There is 1 (0 + 1) difference. Segment 0 changes to 255. The country index is 235, which is "GB" +in the COUNTRIES list. + +It would deserialize into three CountryBlocks: +CountryBlock { // this block is implied by the fact that the first start address wasn't 0.0.0.0 + pub ip_range: IpRange::V6(Ipv6Addr.from_str("0:0:0:0:0:0:0:0").unwrap(), Ipv6Addr.from_str("1:2:3:3:FFFF:FFFF:FFFF:FFFF").unwrap()), + pub country: Country::try_from("ZZ").unwrap(), // generated blocks are always for ZZ +} +CountryBlock { + pub ip_range: IpRange::V6(Ipv6Addr.from_str("1:2:3:4:0:0:0:0").unwrap(), Ipv6Addr.from_str("FF:2:3:3:FFFF:FFFF:FFFF:FFFF").unwrap()), + pub country: Country::try_from("US").unwrap(), +} +CountryBlock { + pub ip_range: IpRange::V6(Ipv6Addr.from_str("FF:2:3:4:0:0:0:0").unwrap(), Ipv6Addr.from_str("FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF").unwrap()), + pub country: Country::try_from("GB").unwrap(), +} + + */ + +type Ipv4Serializer = VersionedIPSerializer; +type Ipv6Serializer = VersionedIPSerializer; + +pub struct FinalBitQueue { + pub bit_queue: BitQueue, + pub block_count: usize, +} + +impl Default for FinalBitQueue { + fn default() -> Self { + Self { + bit_queue: BitQueue::new(), + block_count: 0, + } + } +} + +pub struct CountryBlockSerializer { + ipv4: Ipv4Serializer, + ipv6: Ipv6Serializer, +} + +impl Default for CountryBlockSerializer { + fn default() -> Self { + Self::new() + } +} + +impl CountryBlockSerializer { + pub fn new() -> Self { + Self { + ipv4: VersionedIPSerializer::new( + Ipv4Addr::new(0xFF, 0xFF, 0xFF, 0xFE), + Ipv4Addr::new(0xFF, 0xFF, 0xFF, 0xFF), + ), + ipv6: VersionedIPSerializer::new( + Ipv6Addr::new( + 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFE, + ), + Ipv6Addr::new( + 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, + ), + ), + } + } + + pub fn add(&mut self, country_block: CountryBlock) { + match country_block.ip_range { + IpRange::V4(start, end) => self.ipv4.add_ip(start, end, country_block.country.index), + IpRange::V6(start, end) => self.ipv6.add_ip(start, end, country_block.country.index), + } + } + + pub fn finish(mut self) -> (FinalBitQueue, FinalBitQueue) { + let last_ipv4 = Ipv4Addr::new(0xFF, 0xFF, 0xFF, 0xFF); + let last_ipv6 = Ipv6Addr::new( + 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, + ); + if self.ipv4.prev_end.ip != last_ipv4 { + self.ipv4 + .add_ip(Ipv4Addr::plus_one_ip(self.ipv4.prev_end.ip), last_ipv4, 0); + } + if self.ipv6.prev_end.ip != last_ipv6 { + self.ipv6 + .add_ip(Ipv6Addr::plus_one_ip(self.ipv6.prev_end.ip), last_ipv6, 0); + } + let bit_queue_ipv4 = self.ipv4.bit_queue; + let block_count_ipv4 = self.ipv4.block_count; + let bit_queue_ipv6 = self.ipv6.bit_queue; + let block_count_ipv6 = self.ipv6.block_count; + ( + FinalBitQueue { + bit_queue: bit_queue_ipv4, + block_count: block_count_ipv4, + }, + FinalBitQueue { + bit_queue: bit_queue_ipv6, + block_count: block_count_ipv6, + }, + ) + } +} + +struct VersionedIPSerializer +where + IPType: Debug, + SegmentNumRep: Debug, +{ + prev_start: VersionedIP, + prev_end: VersionedIP, + block_count: usize, + bit_queue: BitQueue, +} + +trait Serializer { + fn add_ip(&mut self, start: IPType, end: IPType, country_index: usize); +} + +impl Serializer for Ipv4Serializer { + fn add_ip(&mut self, start: Ipv4Addr, end: Ipv4Addr, country_index: usize) { + self.add_ip_generic(start, end, country_index, 2, 2, 8) + } +} + +impl Serializer for Ipv6Serializer { + fn add_ip(&mut self, start: Ipv6Addr, end: Ipv6Addr, country_index: usize) { + self.add_ip_generic(start, end, country_index, 3, 3, 16) + } +} + +impl + VersionedIPSerializer +where + IPType: + PlusMinusOneIP + IPIntoSegments + Copy + PartialEq + Debug, + SegmentNumRep: PartialEq + Debug, + u64: From, +{ + fn add_ip_generic( + &mut self, + start: IPType, + end: IPType, + country_index: usize, + difference_count_bit_count: usize, + index_bit_count: usize, + segment_bit_count: usize, + ) { + let expected_start = IPType::plus_one_ip(self.prev_end.ip); + if start != expected_start { + self.add_ip_generic( + expected_start, + IPType::minus_one_ip(start), + 0, + difference_count_bit_count, + index_bit_count, + segment_bit_count, + ) + } + let differences = Self::ips_into_differences(self.prev_start.ip, start); + let difference_count_minus_one = (differences.len() - 1) as u64; + self.bit_queue + .add_bits(difference_count_minus_one, difference_count_bit_count); + differences.into_iter().for_each(|difference| { + self.bit_queue + .add_bits(difference.index as u64, index_bit_count); + self.bit_queue.add_bits(difference.value, segment_bit_count); + }); + self.bit_queue.add_bits(country_index as u64, 9); + self.prev_start.ip = start; + self.prev_end.ip = end; + self.block_count += 1; + } +} + +impl + VersionedIPSerializer +where + IPType: IPIntoSegments + Debug, + SegmentNumRep: PartialEq + Debug, + u64: From, +{ + fn new( + prev_start: IPType, + prev_end: IPType, + ) -> VersionedIPSerializer { + let prev_start = VersionedIP::new(prev_start); + let prev_end = VersionedIP::new(prev_end); + let bit_queue = BitQueue::new(); + Self { + prev_start, + prev_end, + block_count: 0, + bit_queue, + } + } + + fn ips_into_differences(from: IPType, to: IPType) -> Vec { + let pairs = from.segments().into_iter().zip(to.segments().into_iter()); + pairs + .enumerate() + .flat_map( + |(index, (from_segment, to_segment)): (_, (SegmentNumRep, SegmentNumRep))| { + if to_segment == from_segment { + None + } else { + Some(Difference { + index, + value: u64::from(to_segment), + }) + } + }, + ) + .collect() + } +} + +// Rust forces public visibility on traits that come to be used as type boundaries in any public +// interface. This is how we can meet the requirements while the implementations of such +// traits become ineffective beyond this file. It works as a form of prevention to +// namespace pollution for such kind of trait to be implemented on massively common types, +// here namely Ipv4Addr or Ipv6Addr +mod semi_private_items { + use crate::bit_queue::BitQueue; + + pub trait IPIntoSegments { + fn segments(&self) -> [BitsPerSegment; SEGMENTS_COUNT]; + } + + pub trait IPIntoOctets { + fn octets(&self) -> [u8; OCTETS_COUNT]; + } + + pub trait PlusMinusOneIP { + fn plus_one_ip(ip: Self) -> Self; + fn minus_one_ip(ip: Self) -> Self; + } + + pub trait DeserializerPrivate { + fn max_ip_value() -> IPType; + fn read_difference_count(bit_queue: &mut BitQueue) -> Option; + fn read_differences(bit_queue: &mut BitQueue, difference_count: usize) -> Vec; + } + + pub struct Difference { + pub index: usize, + pub value: u64, + } +} + +impl IPIntoSegments for Ipv4Addr { + fn segments(&self) -> [u8; 4] { + self.octets() + } +} + +impl IPIntoSegments for Ipv6Addr { + fn segments(&self) -> [u16; 8] { + self.segments() + } +} + +impl IPIntoOctets<4> for Ipv4Addr { + fn octets(&self) -> [u8; 4] { + self.segments() + } +} + +impl IPIntoOctets<16> for Ipv6Addr { + fn octets(&self) -> [u8; 16] { + self.octets() + } +} + +impl PlusMinusOneIP for Ipv4Addr { + fn plus_one_ip(ip: Self) -> Self { + let old_data: u32 = integer_from_ip_generic(ip); + let new_data = old_data.overflowing_add(1).0; + Ipv4Addr::from(new_data) + } + fn minus_one_ip(ip: Self) -> Self { + let old_data: u32 = integer_from_ip_generic(ip); + let new_data = old_data.overflowing_sub(1).0; + Ipv4Addr::from(new_data) + } +} + +impl PlusMinusOneIP for Ipv6Addr { + fn plus_one_ip(ip: Self) -> Self { + let old_data: u128 = integer_from_ip_generic(ip); + let new_data = old_data.overflowing_add(1).0; + Ipv6Addr::from(new_data) + } + + fn minus_one_ip(ip: Self) -> Self { + let old_data: u128 = integer_from_ip_generic(ip); + let new_data = old_data.overflowing_sub(1).0; + Ipv6Addr::from(new_data) + } +} + +fn integer_from_ip_generic( + ip: IPType, +) -> UnsignedInt +where + IPType: IPIntoOctets, + UnsignedInt: From + BitOrAssign + ShlAssign, +{ + let segments = ip.octets(); + let mut bit_data = UnsignedInt::from(0u8); + segments.into_iter().for_each(|octet| { + bit_data <<= UnsignedInt::from(8u8); + bit_data |= UnsignedInt::from(octet); + }); + bit_data +} + +#[derive(Debug)] +pub struct CountryBlockDeserializer< + 'a, + IPType: Debug, + SegmentNumRep: Debug, + const SEGMENTS_COUNT: usize, +> { + countries: &'a Countries, + prev_record: StreamRecord, + bit_queue: BitQueue, + empty: bool, +} + +pub trait DeserializerPublic { + fn new(country_data: (Vec, usize)) -> Self; + fn next(&mut self) -> Option; +} + +pub type Ipv4CountryBlockDeserializer<'a> = CountryBlockDeserializer<'a, Ipv4Addr, u8, 4>; + +impl<'a> Ipv4CountryBlockDeserializer<'a> { + pub fn new(country_data: (Vec, usize), countries: &'a Countries) -> Self { + Self::new_generic( + country_data, + Ipv4Addr::new(0xFF, 0xFF, 0xFF, 0xFE), + countries, + ) + } +} + +impl<'a> Iterator for Ipv4CountryBlockDeserializer<'a> { + type Item = CountryBlock; + + fn next(&mut self) -> Option { + self.next_generic() + } +} + +pub type Ipv6CountryBlockDeserializer<'a> = CountryBlockDeserializer<'a, Ipv6Addr, u16, 8>; + +impl<'a> Ipv6CountryBlockDeserializer<'a> { + pub fn new(country_data: (Vec, usize), countries: &'a Countries) -> Self { + Self::new_generic( + country_data, + Ipv6Addr::new( + 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFE, + ), + countries, + ) + } +} + +impl<'a> Iterator for Ipv6CountryBlockDeserializer<'a> { + type Item = CountryBlock; + + fn next(&mut self) -> Option { + self.next_generic() + } +} + +impl<'a> DeserializerPrivate for Ipv4CountryBlockDeserializer<'a> { + fn max_ip_value() -> Ipv4Addr { + Ipv4Addr::new(0xFF, 0xFF, 0xFF, 0xFF) + } + + fn read_difference_count(bit_queue: &mut BitQueue) -> Option { + Some((bit_queue.take_bits(2)? + 1) as usize) + } + + fn read_differences(bit_queue: &mut BitQueue, difference_count: usize) -> Vec { + Self::read_differences_generic(bit_queue, difference_count, 2, 8) + } +} + +impl<'a> DeserializerPrivate for Ipv6CountryBlockDeserializer<'a> { + fn max_ip_value() -> Ipv6Addr { + Ipv6Addr::new( + 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, + ) + } + + fn read_difference_count(bit_queue: &mut BitQueue) -> Option { + Some((bit_queue.take_bits(3)? + 1) as usize) + } + + fn read_differences(bit_queue: &mut BitQueue, difference_count: usize) -> Vec { + Self::read_differences_generic(bit_queue, difference_count, 3, 16) + } +} + +impl<'a, IPType, SegmentNumRep, const SEGMENTS_COUNT: usize> + CountryBlockDeserializer<'a, IPType, SegmentNumRep, SEGMENTS_COUNT> +where + Self: DeserializerPrivate, + IPType: IPIntoSegments + + PlusMinusOneIP + + From<[SegmentNumRep; SEGMENTS_COUNT]> + + Copy + + Debug, + SegmentNumRep: TryFrom + Debug, + >::Error: Debug, + IpRange: From<(IPType, IPType)>, +{ + fn new_generic( + country_data: (Vec, usize), + previous_start: IPType, + countries: &'a Countries, + ) -> CountryBlockDeserializer<'a, IPType, SegmentNumRep, SEGMENTS_COUNT> { + let mut bit_queue = bit_queue_from_country_data(country_data); + let prev_record = + CountryBlockDeserializer::::get_record_generic( + &mut bit_queue, + previous_start, + ) + .expect("Empty BitQueue"); + Self { + prev_record, + bit_queue, + empty: false, + countries, + } + } + + fn get_record_generic( + bit_queue: &mut BitQueue, + prev_start: IPType, + ) -> Option> { + let segments: [SegmentNumRep; SEGMENTS_COUNT] = prev_start.segments(); + let difference_count = Self::read_difference_count(bit_queue)?; + let differences = Self::read_differences(bit_queue, difference_count); + if differences.len() < difference_count { + return None; + } + let country_idx = bit_queue.take_bits(9)? as usize; + Some(StreamRecord::::new( + differences, + segments, + country_idx, + )) + } + + fn next_generic(&mut self) -> Option { + if self.empty { + return None; + } + let next_record_opt = + Self::get_record_generic(&mut self.bit_queue, self.prev_record.start.ip); + match next_record_opt { + Some(next_record) => { + let prev_block = CountryBlock { + ip_range: IpRange::from(( + self.prev_record.start.ip, + IPType::minus_one_ip(next_record.start.ip), + )), + country: self + .countries + .country_from_index(self.prev_record.country_idx) + .expect("Country not found") + .clone(), + }; + self.prev_record = next_record; + Some(prev_block) + } + None => { + self.empty = true; + Some(CountryBlock { + ip_range: IpRange::from((self.prev_record.start.ip, Self::max_ip_value())), + country: self + .countries + .country_from_index(self.prev_record.country_idx) + .expect("Country not found") + .clone(), + }) + } + } + } + + fn read_differences_generic( + bit_queue: &mut BitQueue, + difference_count: usize, + index_bit_count: usize, + value_bit_count: usize, + ) -> Vec { + (0..difference_count) + .filter_map(|_| { + Some(Difference { + index: bit_queue.take_bits(index_bit_count)? as usize, + value: bit_queue.take_bits(value_bit_count)?, + }) + }) + .collect() + } +} + +#[derive(Debug)] +struct VersionedIP +where + IPType: Debug, + SegmentNumRep: Debug, +{ + ip: IPType, + segment_num_rep: PhantomData, +} + +impl + VersionedIP +where + IPType: Debug, + SegmentNumRep: Debug, +{ + fn new(ip: IPType) -> VersionedIP { + let segment_num_rep = Default::default(); + Self { + ip, + segment_num_rep, + } + } +} + +#[derive(Debug)] +struct StreamRecord +where + IPType: Debug, + SegmentNumRep: Debug, +{ + start: VersionedIP, + country_idx: usize, +} + +impl + StreamRecord +where + IPType: From<[SegmentNumRep; SEGMENTS_COUNT]> + Debug, + SegmentNumRep: TryFrom + Debug, + ::Error: Debug, +{ + fn new( + differences: Vec, + mut segments: [SegmentNumRep; SEGMENTS_COUNT], + country_idx: usize, + ) -> StreamRecord { + differences.into_iter().for_each(|d| { + segments[d.index] = SegmentNumRep::try_from(d.value).expect( + "Difference represented by a bigger number than which the IP segment can contain", + ) + }); + Self { + start: VersionedIP::new(IPType::from(segments)), + country_idx, + } + } +} + +impl From<(Ipv4Addr, Ipv4Addr)> for IpRange { + fn from((start, end): (Ipv4Addr, Ipv4Addr)) -> Self { + IpRange::V4(start, end) + } +} + +impl From<(Ipv6Addr, Ipv6Addr)> for IpRange { + fn from((start, end): (Ipv6Addr, Ipv6Addr)) -> Self { + IpRange::V6(start, end) + } +} + +fn bit_queue_from_country_data(country_data_pair: (Vec, usize)) -> BitQueue { + let (mut country_data, mut bit_count) = country_data_pair; + let mut bit_queue = BitQueue::new(); + while bit_count >= 64 { + bit_queue.add_bits(country_data.remove(0), 64); + bit_count -= 64; + } + if bit_count > 0 { + bit_queue.add_bits(country_data.remove(0), bit_count); + } + bit_queue +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::countries::Countries; + use crate::country_block_stream::{Country, IpRange}; + use lazy_static::lazy_static; + use std::net::Ipv4Addr; + use std::str::FromStr; + + lazy_static! { + static ref SENTINEL: Country = Country::new(0, "ZZ", "Sentinel"); + static ref ANDORRA: Country = Country::new(1, "AD", "Andorra"); + static ref ANGOLA: Country = Country::new(2, "AO", "Angola"); + static ref AMERICAN_SAMOA: Country = Country::new(3, "AS", "American Samoa"); + static ref TEST_COUNTRIES: Countries = Countries::old_new(vec![ + SENTINEL.clone(), + ANDORRA.clone(), + ANGOLA.clone(), + AMERICAN_SAMOA.clone(), + ]); + } + + fn ipv4_country_blocks() -> Vec { + vec![ + CountryBlock { + ip_range: IpRange::V4( + Ipv4Addr::from_str("1.2.3.4").unwrap(), + Ipv4Addr::from_str("1.2.3.5").unwrap(), + ), + country: AMERICAN_SAMOA.clone(), + }, + CountryBlock { + ip_range: IpRange::V4( + Ipv4Addr::from_str("1.2.3.6").unwrap(), + Ipv4Addr::from_str("6.7.8.9").unwrap(), + ), + country: ANDORRA.clone(), + }, + CountryBlock { + ip_range: IpRange::V4( + Ipv4Addr::from_str("10.11.12.13").unwrap(), + Ipv4Addr::from_str("11.11.12.13").unwrap(), + ), + country: ANGOLA.clone(), + }, + ] + } + + fn ipv6_country_blocks() -> Vec { + vec![ + CountryBlock { + ip_range: IpRange::V6( + Ipv6Addr::from_str("1:2:3:4:5:6:7:8").unwrap(), + Ipv6Addr::from_str("1:2:3:4:5:6:7:9").unwrap(), + ), + country: AMERICAN_SAMOA.clone(), + }, + CountryBlock { + ip_range: IpRange::V6( + Ipv6Addr::from_str("1:2:3:4:5:6:7:A").unwrap(), + Ipv6Addr::from_str("B:C:D:E:F:10:11:12").unwrap(), + ), + country: ANDORRA.clone(), + }, + CountryBlock { + ip_range: IpRange::V6( + Ipv6Addr::from_str("13:14:15:16:17:18:19:1A").unwrap(), + Ipv6Addr::from_str("14:14:15:16:17:18:19:1A").unwrap(), + ), + country: ANGOLA.clone(), + }, + ] + } + + fn expected_ipv4() -> Vec { + let mut country_blocks = ipv4_country_blocks(); + vec![ + CountryBlock { + ip_range: IpRange::V4( + Ipv4Addr::from_str("0.0.0.0").unwrap(), + Ipv4Addr::from_str("1.2.3.3").unwrap(), + ), + country: SENTINEL.clone(), + }, + country_blocks.remove(0), + country_blocks.remove(0), + CountryBlock { + ip_range: IpRange::V4( + Ipv4Addr::from_str("6.7.8.10").unwrap(), + Ipv4Addr::from_str("10.11.12.12").unwrap(), + ), + country: SENTINEL.clone(), + }, + country_blocks.remove(0), + CountryBlock { + ip_range: IpRange::V4( + Ipv4Addr::from_str("11.11.12.14").unwrap(), + Ipv4Addr::from_str("255.255.255.255").unwrap(), + ), + country: SENTINEL.clone(), + }, + ] + } + + fn expected_ipv6() -> Vec { + let mut country_blocks = ipv6_country_blocks(); + vec![ + CountryBlock { + ip_range: IpRange::V6( + Ipv6Addr::from_str("0:0:0:0:0:0:0:0").unwrap(), + Ipv6Addr::from_str("1:2:3:4:5:6:7:7").unwrap(), + ), + country: SENTINEL.clone(), + }, + country_blocks.remove(0), + country_blocks.remove(0), + CountryBlock { + ip_range: IpRange::V6( + Ipv6Addr::from_str("B:C:D:E:F:10:11:13").unwrap(), + Ipv6Addr::from_str("13:14:15:16:17:18:19:19").unwrap(), + ), + country: SENTINEL.clone(), + }, + country_blocks.remove(0), + CountryBlock { + ip_range: IpRange::V6( + Ipv6Addr::from_str("14:14:15:16:17:18:19:1B").unwrap(), + Ipv6Addr::from_str("FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF").unwrap(), + ), + country: SENTINEL.clone(), + }, + ] + } + + #[test] + fn versioned_ip_implements_debug() { + let ip4: VersionedIP = VersionedIP::new(Ipv4Addr::new(1, 2, 3, 4)); + let ip6: VersionedIP = + VersionedIP::new(Ipv6Addr::new(1, 2, 3, 4, 5, 6, 7, 8)); + + let result_ip4 = format!("{:?}", ip4); + let result_ip6 = format!("{:?}", ip6); + + assert_eq!( + result_ip4, + "VersionedIP { ip: 1.2.3.4, segment_num_rep: PhantomData }" + ); + assert_eq!( + result_ip6, + "VersionedIP { ip: 1:2:3:4:5:6:7:8, segment_num_rep: PhantomData }" + ); + } + + #[test] + fn add_works_for_ipv4() { + let mut country_blocks = ipv4_country_blocks(); + let mut subject = CountryBlockSerializer::new(); + + subject.add(country_blocks.remove(0)); + subject.add(country_blocks.remove(0)); + subject.add(country_blocks.remove(0)); + + let (final_ipv4, _) = subject.finish(); + let mut bit_queue = final_ipv4.bit_queue; + { + let ( + difference_count_minus_one, + index1, + value1, + index2, + value2, + index3, + value3, + index4, + value4, + country_index, + ) = ( + bit_queue.take_bits(2).unwrap(), + bit_queue.take_bits(2).unwrap(), + bit_queue.take_bits(8).unwrap(), + bit_queue.take_bits(2).unwrap(), + bit_queue.take_bits(8).unwrap(), + bit_queue.take_bits(2).unwrap(), + bit_queue.take_bits(8).unwrap(), + bit_queue.take_bits(2).unwrap(), + bit_queue.take_bits(8).unwrap(), + bit_queue.take_bits(9).unwrap(), + ); + assert_eq!(difference_count_minus_one, 3); + assert_eq!((index1, value1), (0, 0)); + assert_eq!((index2, value2), (1, 0)); + assert_eq!((index3, value3), (2, 0)); + assert_eq!((index4, value4), (3, 0)); + assert_eq!(country_index as usize, SENTINEL.index) + } + { + let ( + difference_count_minus_one, + index1, + value1, + index2, + value2, + index3, + value3, + index4, + value4, + country_index, + ) = ( + bit_queue.take_bits(2).unwrap(), + bit_queue.take_bits(2).unwrap(), + bit_queue.take_bits(8).unwrap(), + bit_queue.take_bits(2).unwrap(), + bit_queue.take_bits(8).unwrap(), + bit_queue.take_bits(2).unwrap(), + bit_queue.take_bits(8).unwrap(), + bit_queue.take_bits(2).unwrap(), + bit_queue.take_bits(8).unwrap(), + bit_queue.take_bits(9).unwrap(), + ); + assert_eq!(difference_count_minus_one, 3); + assert_eq!((index1, value1), (0, 1)); + assert_eq!((index2, value2), (1, 2)); + assert_eq!((index3, value3), (2, 3)); + assert_eq!((index4, value4), (3, 4)); + assert_eq!(country_index as usize, AMERICAN_SAMOA.index); + } + { + let (difference_count_minus_one, index1, value1, country_index) = ( + bit_queue.take_bits(2).unwrap(), + bit_queue.take_bits(2).unwrap(), + bit_queue.take_bits(8).unwrap(), + bit_queue.take_bits(9).unwrap(), + ); + assert_eq!(difference_count_minus_one, 0); + assert_eq!((index1, value1), (3, 6)); + assert_eq!(country_index as usize, ANDORRA.index); + } + { + let ( + difference_count_minus_one, + index1, + value1, + index2, + value2, + index3, + value3, + index4, + value4, + country_index, + ) = ( + bit_queue.take_bits(2).unwrap(), + bit_queue.take_bits(2).unwrap(), + bit_queue.take_bits(8).unwrap(), + bit_queue.take_bits(2).unwrap(), + bit_queue.take_bits(8).unwrap(), + bit_queue.take_bits(2).unwrap(), + bit_queue.take_bits(8).unwrap(), + bit_queue.take_bits(2).unwrap(), + bit_queue.take_bits(8).unwrap(), + bit_queue.take_bits(9).unwrap(), + ); + assert_eq!(difference_count_minus_one, 3); + assert_eq!((index1, value1), (0, 6)); + assert_eq!((index2, value2), (1, 7)); + assert_eq!((index3, value3), (2, 8)); + assert_eq!((index4, value4), (3, 10)); + assert_eq!(country_index as usize, SENTINEL.index); + } + { + let ( + difference_count_minus_one, + index1, + value1, + index2, + value2, + index3, + value3, + index4, + value4, + country_index, + ) = ( + bit_queue.take_bits(2).unwrap(), + bit_queue.take_bits(2).unwrap(), + bit_queue.take_bits(8).unwrap(), + bit_queue.take_bits(2).unwrap(), + bit_queue.take_bits(8).unwrap(), + bit_queue.take_bits(2).unwrap(), + bit_queue.take_bits(8).unwrap(), + bit_queue.take_bits(2).unwrap(), + bit_queue.take_bits(8).unwrap(), + bit_queue.take_bits(9).unwrap(), + ); + assert_eq!(difference_count_minus_one, 3); + assert_eq!((index1, value1), (0, 10)); + assert_eq!((index2, value2), (1, 11)); + assert_eq!((index3, value3), (2, 12)); + assert_eq!((index4, value4), (3, 13)); + assert_eq!(country_index as usize, ANGOLA.index); + } + { + let (difference_count_minus_one, index1, value1, index2, value2, country_index) = ( + bit_queue.take_bits(2).unwrap(), + bit_queue.take_bits(2).unwrap(), + bit_queue.take_bits(8).unwrap(), + bit_queue.take_bits(2).unwrap(), + bit_queue.take_bits(8).unwrap(), + bit_queue.take_bits(9).unwrap(), + ); + assert_eq!(difference_count_minus_one, 1); + assert_eq!((index1, value1), (0, 11)); + assert_eq!((index2, value2), (3, 14)); + assert_eq!(country_index as usize, SENTINEL.index); + } + assert_eq!(bit_queue.take_bits(1), None); + } + + #[test] + fn next_works_for_ipv4() { + let mut serializer = CountryBlockSerializer::new(); + ipv4_country_blocks() + .into_iter() + .for_each(|country_block| serializer.add(country_block)); + let (final_ipv4, _) = serializer.finish(); + let mut bit_queue = final_ipv4.bit_queue; + let bit_queue_len = bit_queue.len(); + let mut bit_data: Vec = vec![]; + while bit_queue.len() >= 64 { + let data = bit_queue.take_bits(64).unwrap(); + bit_data.push(data); + } + let remaining_bit_count = bit_queue.len(); + let data = bit_queue.take_bits(remaining_bit_count).unwrap(); + bit_data.push(data); + let mut subject = CountryBlockDeserializer::::new( + (bit_data, bit_queue_len), + &TEST_COUNTRIES, + ); + let mut actual_country_blocks: Vec = vec![]; + loop { + match subject.next() { + None => break, + Some(country_block) => actual_country_blocks.push(country_block), + } + } + + assert_eq!(actual_country_blocks, expected_ipv4()); + } + + #[test] + fn finish_does_not_touch_complete_ipv4_list() { + // let mut country_blocks = ipv4_country_blocks(); + let mut subject = CountryBlockSerializer::new(); + expected_ipv4().into_iter().for_each(|cb| subject.add(cb)); + let (final_ipv4, _) = subject.finish(); + let mut bitqueue = final_ipv4.bit_queue; + let len = bitqueue.len(); + let mut vec_64 = vec![]; + while bitqueue.len() >= 64 { + let data = bitqueue.take_bits(64).unwrap(); + vec_64.push(data); + } + let remaining_bit_count = bitqueue.len(); + let data = bitqueue.take_bits(remaining_bit_count).unwrap(); + vec_64.push(data); + + let mut deserializer = + CountryBlockDeserializer::::new((vec_64, len), &TEST_COUNTRIES); + + let mut country_blocks: Vec = vec![]; + loop { + match deserializer.next() { + None => break, + Some(country_block) => country_blocks.push(country_block), + } + } + let iso3166s = country_blocks + .into_iter() + .map(|cb| cb.country.iso3166) + .collect::>() + .join(", "); + assert_eq!(iso3166s, "ZZ, AS, AD, ZZ, AO, ZZ".to_string()); + } + + #[test] + fn add_works_for_ipv6() { + let mut country_blocks = ipv6_country_blocks(); + let mut subject = CountryBlockSerializer::new(); + + subject.add(country_blocks.remove(0)); + subject.add(country_blocks.remove(0)); + subject.add(country_blocks.remove(0)); + + let (_, final_ipv6) = subject.finish(); + let mut bit_queue = final_ipv6.bit_queue; + { + let ( + difference_count_minus_one, + index1, + value1, + index2, + value2, + index3, + value3, + index4, + value4, + index5, + value5, + index6, + value6, + index7, + value7, + index8, + value8, + country_index, + ) = ( + bit_queue.take_bits(3).unwrap(), + bit_queue.take_bits(3).unwrap(), + bit_queue.take_bits(16).unwrap(), + bit_queue.take_bits(3).unwrap(), + bit_queue.take_bits(16).unwrap(), + bit_queue.take_bits(3).unwrap(), + bit_queue.take_bits(16).unwrap(), + bit_queue.take_bits(3).unwrap(), + bit_queue.take_bits(16).unwrap(), + bit_queue.take_bits(3).unwrap(), + bit_queue.take_bits(16).unwrap(), + bit_queue.take_bits(3).unwrap(), + bit_queue.take_bits(16).unwrap(), + bit_queue.take_bits(3).unwrap(), + bit_queue.take_bits(16).unwrap(), + bit_queue.take_bits(3).unwrap(), + bit_queue.take_bits(16).unwrap(), + bit_queue.take_bits(9).unwrap(), + ); + assert_eq!(difference_count_minus_one, 7); + assert_eq!((index1, value1), (0, 0)); + assert_eq!((index2, value2), (1, 0)); + assert_eq!((index3, value3), (2, 0)); + assert_eq!((index4, value4), (3, 0)); + assert_eq!((index5, value5), (4, 0)); + assert_eq!((index6, value6), (5, 0)); + assert_eq!((index7, value7), (6, 0)); + assert_eq!((index8, value8), (7, 0)); + assert_eq!(country_index as usize, SENTINEL.index) + } + { + let ( + difference_count_minus_one, + index1, + value1, + index2, + value2, + index3, + value3, + index4, + value4, + index5, + value5, + index6, + value6, + index7, + value7, + index8, + value8, + country_index, + ) = ( + bit_queue.take_bits(3).unwrap(), + bit_queue.take_bits(3).unwrap(), + bit_queue.take_bits(16).unwrap(), + bit_queue.take_bits(3).unwrap(), + bit_queue.take_bits(16).unwrap(), + bit_queue.take_bits(3).unwrap(), + bit_queue.take_bits(16).unwrap(), + bit_queue.take_bits(3).unwrap(), + bit_queue.take_bits(16).unwrap(), + bit_queue.take_bits(3).unwrap(), + bit_queue.take_bits(16).unwrap(), + bit_queue.take_bits(3).unwrap(), + bit_queue.take_bits(16).unwrap(), + bit_queue.take_bits(3).unwrap(), + bit_queue.take_bits(16).unwrap(), + bit_queue.take_bits(3).unwrap(), + bit_queue.take_bits(16).unwrap(), + bit_queue.take_bits(9).unwrap(), + ); + assert_eq!(difference_count_minus_one, 7); + assert_eq!((index1, value1), (0, 1)); + assert_eq!((index2, value2), (1, 2)); + assert_eq!((index3, value3), (2, 3)); + assert_eq!((index4, value4), (3, 4)); + assert_eq!((index5, value5), (4, 5)); + assert_eq!((index6, value6), (5, 6)); + assert_eq!((index7, value7), (6, 7)); + assert_eq!((index8, value8), (7, 8)); + assert_eq!(country_index as usize, AMERICAN_SAMOA.index); + } + { + let (difference_count_minus_one, index1, value1, country_index) = ( + bit_queue.take_bits(3).unwrap(), + bit_queue.take_bits(3).unwrap(), + bit_queue.take_bits(16).unwrap(), + bit_queue.take_bits(9).unwrap(), + ); + assert_eq!(difference_count_minus_one, 0); + assert_eq!((index1, value1), (7, 10)); + assert_eq!(country_index as usize, ANDORRA.index); + } + { + let ( + difference_count_minus_one, + index1, + value1, + index2, + value2, + index3, + value3, + index4, + value4, + index5, + value5, + index6, + value6, + index7, + value7, + index8, + value8, + country_index, + ) = ( + bit_queue.take_bits(3).unwrap(), + bit_queue.take_bits(3).unwrap(), + bit_queue.take_bits(16).unwrap(), + bit_queue.take_bits(3).unwrap(), + bit_queue.take_bits(16).unwrap(), + bit_queue.take_bits(3).unwrap(), + bit_queue.take_bits(16).unwrap(), + bit_queue.take_bits(3).unwrap(), + bit_queue.take_bits(16).unwrap(), + bit_queue.take_bits(3).unwrap(), + bit_queue.take_bits(16).unwrap(), + bit_queue.take_bits(3).unwrap(), + bit_queue.take_bits(16).unwrap(), + bit_queue.take_bits(3).unwrap(), + bit_queue.take_bits(16).unwrap(), + bit_queue.take_bits(3).unwrap(), + bit_queue.take_bits(16).unwrap(), + bit_queue.take_bits(9).unwrap(), + ); + assert_eq!(difference_count_minus_one, 7); + assert_eq!((index1, value1), (0, 0xB)); + assert_eq!((index2, value2), (1, 0xC)); + assert_eq!((index3, value3), (2, 0xD)); + assert_eq!((index4, value4), (3, 0xE)); + assert_eq!((index5, value5), (4, 0xF)); + assert_eq!((index6, value6), (5, 0x10)); + assert_eq!((index7, value7), (6, 0x11)); + assert_eq!((index8, value8), (7, 0x13)); + assert_eq!(country_index as usize, SENTINEL.index); + } + { + let ( + difference_count_minus_one, + index1, + value1, + index2, + value2, + index3, + value3, + index4, + value4, + index5, + value5, + index6, + value6, + index7, + value7, + index8, + value8, + country_index, + ) = ( + bit_queue.take_bits(3).unwrap(), + bit_queue.take_bits(3).unwrap(), + bit_queue.take_bits(16).unwrap(), + bit_queue.take_bits(3).unwrap(), + bit_queue.take_bits(16).unwrap(), + bit_queue.take_bits(3).unwrap(), + bit_queue.take_bits(16).unwrap(), + bit_queue.take_bits(3).unwrap(), + bit_queue.take_bits(16).unwrap(), + bit_queue.take_bits(3).unwrap(), + bit_queue.take_bits(16).unwrap(), + bit_queue.take_bits(3).unwrap(), + bit_queue.take_bits(16).unwrap(), + bit_queue.take_bits(3).unwrap(), + bit_queue.take_bits(16).unwrap(), + bit_queue.take_bits(3).unwrap(), + bit_queue.take_bits(16).unwrap(), + bit_queue.take_bits(9).unwrap(), + ); + assert_eq!(difference_count_minus_one, 7); + assert_eq!((index1, value1), (0, 0x13)); + assert_eq!((index2, value2), (1, 0x14)); + assert_eq!((index3, value3), (2, 0x15)); + assert_eq!((index4, value4), (3, 0x16)); + assert_eq!((index5, value5), (4, 0x17)); + assert_eq!((index6, value6), (5, 0x18)); + assert_eq!((index7, value7), (6, 0x19)); + assert_eq!((index8, value8), (7, 0x1A)); + assert_eq!(country_index as usize, ANGOLA.index); + } + { + let (difference_count_minus_one, index1, value1, index2, value2, country_index) = ( + bit_queue.take_bits(3).unwrap(), + bit_queue.take_bits(3).unwrap(), + bit_queue.take_bits(16).unwrap(), + bit_queue.take_bits(3).unwrap(), + bit_queue.take_bits(16).unwrap(), + bit_queue.take_bits(9).unwrap(), + ); + assert_eq!(difference_count_minus_one, 1); + assert_eq!((index1, value1), (0, 0x14)); + assert_eq!((index2, value2), (7, 0x1B)); + assert_eq!(country_index as usize, SENTINEL.index); + } + assert_eq!(bit_queue.take_bits(1), None); + } + + #[test] + fn next_works_for_ipv6() { + let mut serializer = CountryBlockSerializer::new(); + ipv6_country_blocks() + .into_iter() + .for_each(|country_block| serializer.add(country_block)); + let (_, final_ipv6) = serializer.finish(); + let mut bit_queue = final_ipv6.bit_queue; + let bit_queue_len = bit_queue.len(); + let mut bit_data: Vec = vec![]; + while bit_queue.len() >= 64 { + let data = bit_queue.take_bits(64).unwrap(); + bit_data.push(data); + } + let remaining_bit_count = bit_queue.len(); + let data = bit_queue.take_bits(remaining_bit_count).unwrap(); + bit_data.push(data); + let mut subject = CountryBlockDeserializer::::new( + (bit_data, bit_queue_len), + &TEST_COUNTRIES, + ); + + let country_block1 = subject.next().unwrap(); + let country_block2 = subject.next().unwrap(); + let country_block3 = subject.next().unwrap(); + let country_block4 = subject.next().unwrap(); + let country_block5 = subject.next().unwrap(); + let country_block6 = subject.next().unwrap(); + let result = subject.next(); + + let original_country_blocks = ipv6_country_blocks(); + assert_eq!( + country_block1, + CountryBlock { + ip_range: IpRange::V6( + Ipv6Addr::from_str("0:0:0:0:0:0:0:0").unwrap(), + Ipv6Addr::from_str("1:2:3:4:5:6:7:7").unwrap() + ), + country: SENTINEL.clone(), + } + ); + assert_eq!(country_block2, original_country_blocks[0]); + assert_eq!(country_block3, original_country_blocks[1]); + assert_eq!( + country_block4, + CountryBlock { + ip_range: IpRange::V6( + Ipv6Addr::from_str("B:C:D:E:F:10:11:13").unwrap(), + Ipv6Addr::from_str("13:14:15:16:17:18:19:19").unwrap(), + ), + country: SENTINEL.clone(), + } + ); + assert_eq!(country_block5, original_country_blocks[2]); + assert_eq!( + country_block6, + CountryBlock { + ip_range: IpRange::V6( + Ipv6Addr::from_str("14:14:15:16:17:18:19:1B").unwrap(), + Ipv6Addr::from_str("FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF").unwrap(), + ), + country: SENTINEL.clone(), + } + ); + assert_eq!(result, None); + } + + #[test] + fn finish_does_not_touch_complete_ipv6_list() { + let mut subject = CountryBlockSerializer::new(); + expected_ipv6().into_iter().for_each(|cb| subject.add(cb)); + let (_, final_ipv6) = subject.finish(); + let mut bitqueue = final_ipv6.bit_queue; + let len = bitqueue.len(); + let mut vec_64 = vec![]; + while bitqueue.len() >= 64 { + let data = bitqueue.take_bits(64).unwrap(); + vec_64.push(data); + } + let remaining_bit_count = bitqueue.len(); + let data = bitqueue.take_bits(remaining_bit_count).unwrap(); + vec_64.push(data); + + let mut deserializer = + CountryBlockDeserializer::::new((vec_64, len), &TEST_COUNTRIES); + + let result = deserializer.next(); + assert_eq!(result.unwrap().country.iso3166, "ZZ"); + let result = deserializer.next(); + assert_eq!(result.unwrap().country.iso3166, "AS"); + let result = deserializer.next(); + assert_eq!(result.unwrap().country.iso3166, "AD"); + let result = deserializer.next(); + assert_eq!(result.unwrap().country.iso3166, "ZZ"); + let result = deserializer.next(); + assert_eq!(result.unwrap().country.iso3166, "AO"); + let result = deserializer.next(); + assert_eq!(result.unwrap().country.iso3166, "ZZ"); + let result = deserializer.next(); + assert_eq!(result, None); + } +} diff --git a/ip_country/src/country_block_stream.rs b/ip_country/src/country_block_stream.rs new file mode 100644 index 000000000..17430f374 --- /dev/null +++ b/ip_country/src/country_block_stream.rs @@ -0,0 +1,235 @@ +// Copyright (c) 2024, MASQ (https://masq.ai) and/or its affiliates. All rights reserved. + +use std::cmp::Ordering; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; + +#[derive(Clone, PartialEq, Debug, Eq)] +pub struct Country { + pub index: usize, + pub iso3166: String, + pub name: String, +} + +impl Country { + pub fn new(index: usize, iso3166: &str, name: &str) -> Self { + Self { + index, + iso3166: iso3166.to_string(), + name: name.to_string(), + } + } +} + +#[derive(Clone, PartialEq, Eq, Debug)] +pub enum IpRange { + V4(Ipv4Addr, Ipv4Addr), + V6(Ipv6Addr, Ipv6Addr), +} + +impl IpRange { + pub fn new(start: IpAddr, end: IpAddr) -> Self { + match (start, end) { + (IpAddr::V4(start), IpAddr::V4(end)) => IpRange::V4(start, end), + (IpAddr::V6(start), IpAddr::V6(end)) => IpRange::V6(start, end), + (start, end) => panic!( + "Start and end addresses must be of the same type, not {} and {}", + start, end + ), + } + } + + pub fn contains(&self, ip_addr: IpAddr) -> bool { + match self { + IpRange::V4(begin, end) => match ip_addr { + IpAddr::V4(candidate) => Self::contains_inner( + u32::from(*begin) as u128, + u32::from(*end) as u128, + u32::from(candidate) as u128, + ), + IpAddr::V6(_candidate) => false, + }, + IpRange::V6(begin, end) => match ip_addr { + IpAddr::V4(_candidate) => false, + IpAddr::V6(candidate) => Self::contains_inner( + u128::from(*begin), + u128::from(*end), + u128::from(candidate), + ), + }, + } + } + + pub fn start(&self) -> IpAddr { + match self { + IpRange::V4(start, _) => IpAddr::V4(*start), + IpRange::V6(start, _) => IpAddr::V6(*start), + } + } + + pub fn end(&self) -> IpAddr { + match self { + IpRange::V4(_, end) => IpAddr::V4(*end), + IpRange::V6(_, end) => IpAddr::V6(*end), + } + } + + pub fn ordering_by_range(&self, ip_addr: IpAddr) -> Ordering { + match (ip_addr, self) { + (IpAddr::V4(ip), IpRange::V4(low, high)) => { + Self::compare_with_range::(ip, *low, *high) + } + (IpAddr::V6(ip), IpRange::V6(low, high)) => { + Self::compare_with_range::(ip, *low, *high) + } + (ip, range) => panic!("Mismatch ip ({}) and range ({:?}) versions", ip, range), + } + } + + fn compare_with_range(examined: IP, low: IP, high: IP) -> Ordering + where + SingleIntegerIPRep: From + PartialOrd, + { + let (low_end, high_end) = ( + SingleIntegerIPRep::from(low), + SingleIntegerIPRep::from(high), + ); + let ip_num = SingleIntegerIPRep::from(examined); + if ip_num < low_end { + Ordering::Greater + } else if ip_num > high_end { + Ordering::Less + } else { + Ordering::Equal + } + } + + fn contains_inner(begin: u128, end: u128, candidate: u128) -> bool { + (begin <= candidate) && (candidate <= end) + } +} + +pub fn are_consecutive(first: IpAddr, second: IpAddr) -> bool { + match (first, second) { + (IpAddr::V4(first), IpAddr::V4(second)) => { + let first = u32::from(first); + let second = u32::from(second); + second == first + 1 + } + (IpAddr::V6(first), IpAddr::V6(second)) => { + let first = u128::from(first); + let second = u128::from(second); + second == first + 1 + } + (first, second) => panic!( + "IP addresses must be of the same type, not {} and {}", + first, second + ), + } +} + +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct CountryBlock { + pub ip_range: IpRange, + pub country: Country, +} + +#[cfg(test)] +mod tests { + use super::*; + use std::net::Ipv4Addr; + use std::str::FromStr; + + #[test] + fn ip_range_finds_ipv4_address() { + let subject = IpRange::V4( + Ipv4Addr::from_str("1.2.3.4").unwrap(), + Ipv4Addr::from_str("4.3.2.1").unwrap(), + ); + + let result_start = subject.contains(IpAddr::from_str("1.2.3.4").unwrap()); + let result_end = subject.contains(IpAddr::from_str("4.3.2.1").unwrap()); + + assert_eq!(result_start, true); + assert_eq!(result_end, true); + } + + #[test] + fn ip_range_doesnt_find_ipv4_address() { + let subject = IpRange::V4( + Ipv4Addr::from_str("1.2.3.4").unwrap(), + Ipv4Addr::from_str("4.3.2.1").unwrap(), + ); + + let result_start = subject.contains(IpAddr::from_str("1.2.3.3").unwrap()); + let result_end = subject.contains(IpAddr::from_str("4.3.2.2").unwrap()); + + assert_eq!(result_start, false); + assert_eq!(result_end, false); + } + + #[test] + fn ip_range_finds_ipv6_address() { + let subject = IpRange::V6( + Ipv6Addr::from_str("1:2:3:4:0:0:0:0").unwrap(), + Ipv6Addr::from_str("4:3:2:1:0:0:0:0").unwrap(), + ); + + let result_start = subject.contains(IpAddr::from_str("1:2:3:4:0:0:0:0").unwrap()); + let result_end = subject.contains(IpAddr::from_str("4:3:2:1:0:0:0:0").unwrap()); + + assert_eq!(result_start, true); + assert_eq!(result_end, true); + } + + #[test] + fn ip_range_doesnt_find_ipv6_address() { + let subject = IpRange::V6( + Ipv6Addr::from_str("0:0:0:0:1:2:3:4").unwrap(), + Ipv6Addr::from_str("0:0:0:0:4:3:2:1").unwrap(), + ); + + let result_start = subject.contains(IpAddr::from_str("0:0:0:0:1:2:3:3").unwrap()); + let result_end = subject.contains(IpAddr::from_str("0:0:0:0:4:3:2:2").unwrap()); + + assert_eq!(result_start, false); + assert_eq!(result_end, false); + } + + #[test] + fn ip_range_doesnt_find_ipv6_address_in_ipv4_range() { + let subject = IpRange::V4( + Ipv4Addr::from_str("1.2.3.4").unwrap(), + Ipv4Addr::from_str("4.3.2.1").unwrap(), + ); + + let result = subject.contains(IpAddr::from_str("1:2:3:4:0:0:0:0").unwrap()); + + assert_eq!(result, false); + } + + #[test] + fn ip_range_doesnt_find_ipv4_address_in_ipv6_range() { + let subject = IpRange::V6( + Ipv6Addr::from_str("0:0:0:0:1:2:3:4").unwrap(), + Ipv6Addr::from_str("0:0:0:0:4:3:2:1").unwrap(), + ); + + let result = subject.contains(IpAddr::from_str("1.2.3.4").unwrap()); + + assert_eq!(result, false); + } + + #[test] + #[should_panic( + expected = "Mismatch ip (1.2.3.4) and range (V6(::1:2:3:4, ::4:3:2:1)) versions" + )] + fn ip_range_panics_on_v4_v6_mismatch() { + let subject = IpRange::V6( + Ipv6Addr::from_str("0:0:0:0:1:2:3:4").unwrap(), + Ipv6Addr::from_str("0:0:0:0:4:3:2:1").unwrap(), + ); + let ip = Ipv4Addr::from_str("1.2.3.4").unwrap(); + + let _result = subject.ordering_by_range(IpAddr::V4(ip)); + } +} diff --git a/ip_country/src/country_finder.rs b/ip_country/src/country_finder.rs new file mode 100644 index 000000000..1309df698 --- /dev/null +++ b/ip_country/src/country_finder.rs @@ -0,0 +1,304 @@ +// Copyright (c) 2024, MASQ (https://masq.ai) and/or its affiliates. All rights reserved. + +use crate::countries::Countries; +use crate::country_block_serde::CountryBlockDeserializer; +use crate::country_block_stream::{Country, CountryBlock}; +use crate::dbip_country; +use itertools::Itertools; +use lazy_static::lazy_static; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; + +lazy_static! { + pub static ref COUNTRY_CODE_FINDER: CountryCodeFinder<'static> = CountryCodeFinder::new( + &dbip_country::COUNTRIES, + dbip_country::ipv4_country_data(), + dbip_country::ipv6_country_data() + ); +} + +pub struct CountryCodeFinder<'a> { + pub countries: &'a Countries, + pub ipv4: Vec, + pub ipv6: Vec, +} + +impl<'a> CountryCodeFinder<'a> { + pub fn new( + countries: &'a Countries, + ipv4_data: (Vec, usize), + ipv6_data: (Vec, usize), + ) -> Self { + Self { + countries, + ipv4: CountryBlockDeserializer::::new(ipv4_data, countries) + .into_iter() + .collect_vec(), + ipv6: CountryBlockDeserializer::::new(ipv6_data, countries) + .into_iter() + .collect_vec(), + } + } + + pub fn find_country(&'a self, ip_addr: IpAddr) -> Option<&'a Country> { + let country_blocks: &[CountryBlock] = match ip_addr { + IpAddr::V4(_) => self.ipv4.as_slice(), + IpAddr::V6(_) => self.ipv6.as_slice(), + }; + let block_index = + country_blocks.binary_search_by(|block| block.ip_range.ordering_by_range(ip_addr)); + let country = match block_index { + Ok(index) => &country_blocks[index].country, + _ => self + .countries + .country_from_code("ZZ") + .expect("expected Country"), + }; + match country.iso3166.as_str() { + "ZZ" => None, + _ => Some(country), + } + } + + pub fn ensure_init(&self) { + //This should provoke lazy_static to perform the value initialization + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::country_block_serde::{Ipv4CountryBlockDeserializer, Ipv6CountryBlockDeserializer}; + use crate::country_block_stream::IpRange; + use crate::dbip_country; + use crate::dbip_country::COUNTRIES; + use std::str::FromStr; + use std::time::SystemTime; + + fn select_country(countries: &Countries, percentage: u8) -> &Country { + // 0% is the first country; 100% is the last country + let country_count_f = countries.len() as f64; + let percentage_f = percentage as f64; + let index = (percentage_f * country_count_f / 101.0).trunc() as usize; + countries.country_from_index(index).unwrap() + } + + #[test] + fn finds_ipv4_address() { + COUNTRY_CODE_FINDER.ensure_init(); + let country = select_country(&COUNTRIES, 50); + let ip_range = &COUNTRY_CODE_FINDER + .ipv4 + .iter() + .find(|block| block.country == *country) + .unwrap() + .ip_range; + let input_ip = match ip_range { + IpRange::V4(start, _) => IpAddr::V4(start.clone()), + _ => panic!("Expected IPv4"), + }; + + let result = CountryCodeFinder::find_country(&COUNTRY_CODE_FINDER, input_ip).unwrap(); + + assert_eq!(result, country); + } + + #[test] + fn does_not_find_ipv4_address_in_zz_block() { + COUNTRY_CODE_FINDER.ensure_init(); + let ip_range = &COUNTRY_CODE_FINDER + .ipv4 + .iter() + .find(|block| &block.country.iso3166 == "ZZ") + .unwrap() + .ip_range; + let input_ip = match ip_range { + IpRange::V4(start, _) => IpAddr::V4(start.clone()), + _ => panic!("Expected IPv4"), + }; + + let result = CountryCodeFinder::find_country(&COUNTRY_CODE_FINDER, input_ip); + + assert_eq!(result, None); + } + + #[test] + fn finds_ipv6_address() { + COUNTRY_CODE_FINDER.ensure_init(); + let country = select_country(&COUNTRIES, 50); + let ip_range = &COUNTRY_CODE_FINDER + .ipv6 + .iter() + .find(|block| block.country == *country) + .unwrap() + .ip_range; + let input_ip = match ip_range { + IpRange::V6(start, _) => IpAddr::V6(start.clone()), + _ => panic!("Expected IPv6"), + }; + + let result = CountryCodeFinder::find_country(&COUNTRY_CODE_FINDER, input_ip).unwrap(); + + assert_eq!(result, country); + } + + #[test] + fn does_not_find_ipv6_address_in_zz_block() { + COUNTRY_CODE_FINDER.ensure_init(); + let ip_range = &COUNTRY_CODE_FINDER + .ipv6 + .iter() + .find(|block| &block.country.iso3166 == "ZZ") + .unwrap() + .ip_range; + let input_ip = match ip_range { + IpRange::V6(start, _) => IpAddr::V6(start.clone()), + _ => panic!("Expected IPv6"), + }; + + let result = CountryCodeFinder::find_country(&COUNTRY_CODE_FINDER, input_ip); + + assert_eq!(result, None) + } + + #[test] + fn real_test_ipv4_with_google() { + if dbip_country::COUNTRIES.country_from_code("US").is_err() { + eprintln!("Country data must be generated (see ip_country/main.rs) in dbip_country.rs before this test will work"); + return; + } + let result = CountryCodeFinder::find_country( + &COUNTRY_CODE_FINDER, + IpAddr::from_str("142.250.191.132").unwrap(), // dig www.google.com A + ) + .unwrap(); + + assert_eq!(result.iso3166, "US".to_string()); + assert_eq!(result.name, "United States".to_string()); + } + + #[test] + fn real_test_ipv4_with_cz_ip() { + if dbip_country::COUNTRIES.country_from_code("CZ").is_err() { + eprintln!("Country data must be generated (see ip_country/main.rs) in dbip_country.rs before this test will work"); + return; + } + let result = CountryCodeFinder::find_country( + &COUNTRY_CODE_FINDER, + IpAddr::from_str("77.75.77.222").unwrap(), // dig www.seznam.cz A + ) + .unwrap(); + + assert_eq!(result.iso3166, "CZ".to_string()); + assert_eq!(result.name, "Czechia".to_string()); + } + + #[test] + fn real_test_ipv4_with_sk_ip() { + if dbip_country::COUNTRIES.country_from_code("SK").is_err() { + eprintln!("Country data must be generated (see ip_country/main.rs) in dbip_country.rs before this test will work"); + return; + } + let time_start = SystemTime::now(); + + let result = CountryCodeFinder::find_country( + &COUNTRY_CODE_FINDER, + IpAddr::from_str("213.81.185.100").unwrap(), // dig www.zoznam.sk A + ) + .unwrap(); + + let time_end = SystemTime::now(); + let duration = time_end.duration_since(time_start).unwrap(); + + assert_eq!(result.iso3166, "SK".to_string()); + assert_eq!(result.name, "Slovakia".to_string()); + assert!( + duration.as_millis() < 1, + "Duration of the search was too long: {} millisecond", + duration.as_millis() + ); + } + + #[test] + fn real_test_ipv6_with_google() { + if dbip_country::COUNTRIES.country_from_code("US").is_err() { + eprintln!("Country data must be generated (see ip_country/main.rs) in dbip_country.rs before this test will work"); + return; + } + let time_start = SystemTime::now(); + + let result = CountryCodeFinder::find_country( + &COUNTRY_CODE_FINDER, + IpAddr::from_str("2607:f8b0:4009:814::2004").unwrap(), // dig www.google.com AAAA + ) + .unwrap(); + + let time_end = SystemTime::now(); + let duration = time_end.duration_since(time_start).unwrap(); + + assert_eq!(result.iso3166, "US".to_string()); + assert_eq!(result.name, "United States".to_string()); + assert!( + duration.as_millis() < 1, + "Duration of the search was too long: {} ms", + duration.as_millis() + ); + } + + #[test] + fn country_blocks_for_ipv4_and_ipv6_are_deserialized_and_inserted_into_vecs() { + let time_start = SystemTime::now(); + + let deserializer_ipv4 = Ipv4CountryBlockDeserializer::new( + dbip_country::ipv4_country_data(), + &dbip_country::COUNTRIES, + ); + let deserializer_ipv6 = Ipv6CountryBlockDeserializer::new( + dbip_country::ipv6_country_data(), + &dbip_country::COUNTRIES, + ); + + let time_end = SystemTime::now(); + let time_start_fill = SystemTime::now(); + + let country_block_finder_ipv4 = deserializer_ipv4.collect_vec(); + let country_block_finder_ipv6 = deserializer_ipv6.collect_vec(); + + let time_end_fill = SystemTime::now(); + let duration_deserialize = time_end.duration_since(time_start).unwrap(); + let duration_fill = time_end_fill.duration_since(time_start_fill).unwrap(); + + assert_eq!( + country_block_finder_ipv4.len(), + dbip_country::ipv4_country_block_count() + ); + assert_eq!( + country_block_finder_ipv6.len(), + dbip_country::ipv6_country_block_count() + ); + assert!( + duration_deserialize.as_secs() < 15, + "Duration of the deserialization was too long: {} ms", + duration_deserialize.as_millis() + ); + assert!( + duration_fill.as_secs() < 8, + "Duration of the filling the vectors was too long: {} ms", + duration_fill.as_millis() + ); + } + + #[test] + fn check_ipv4_ipv6_country_blocks_length() { + let country_block_len_ipv4 = COUNTRY_CODE_FINDER.ipv4.len(); + let country_block_len_ipv6 = COUNTRY_CODE_FINDER.ipv6.len(); + + assert_eq!( + country_block_len_ipv4, + dbip_country::ipv4_country_block_count() + ); + assert_eq!( + country_block_len_ipv6, + dbip_country::ipv6_country_block_count() + ); + } +} diff --git a/ip_country/src/dbip_country.rs b/ip_country/src/dbip_country.rs new file mode 100644 index 000000000..75325eeda --- /dev/null +++ b/ip_country/src/dbip_country.rs @@ -0,0 +1,45 @@ + +use lazy_static::lazy_static; +use crate::countries::Countries; + +lazy_static! { + pub static ref COUNTRIES: Countries = Countries::new( + vec![ + ("ZZ", "Sentinel"), + ("AD", "Andorra"), + ("AO", "Angola"), + ("AS", "American Samoa"), + ] + .into_iter() + .map(|(code, name)| (code.to_string(), name.to_string())) + .collect::>() + ); +} + +pub fn ipv4_country_data() -> (Vec, usize) { + ( + vec![ + 0x8098000300801003, 0x18081B0020981C04, 0xB428C00158440E83, 0x00076162030DC320, + ], + 256 + ) +} + +pub fn ipv4_country_block_count() -> usize { + 6 +} + +pub fn ipv6_country_data() -> (Vec, usize) { + ( + vec![ + 0x3000040000400007, 0x00C0001400020000, 0x4400047000000700, 0x0160002300034000, + 0x800470007C000D40, 0x200163808002B800, 0x1F000398006A000C, 0x004F8008E0010A00, + 0x0AA0014200263800, 0x0018A002F0005980, 0x028080C006B800CE, 0x0000000000001BE0, + ], + 737 + ) +} + +pub fn ipv6_country_block_count() -> usize { + 6 +} diff --git a/ip_country/src/ip_country.rs b/ip_country/src/ip_country.rs new file mode 100644 index 000000000..71fc39b7f --- /dev/null +++ b/ip_country/src/ip_country.rs @@ -0,0 +1,524 @@ +// Copyright (c) 2024, MASQ (https://masq.ai) and/or its affiliates. All rights reserved. + +use crate::bit_queue::BitQueue; +use crate::countries::Countries; +use crate::country_block_serde::FinalBitQueue; +use crate::ip_country_csv::CSVParser; +use crate::ip_country_mmdb::MMDBParser; +use std::any::Any; +use std::io; + +const COUNTRY_BLOCK_BIT_SIZE: usize = 64; + +pub fn ip_country( + args: Vec, + stdin: &mut dyn io::Read, + stdout: &mut dyn io::Write, + stderr: &mut dyn io::Write, + parser_factory: &dyn DBIPParserFactory, +) -> i32 { + let parser = parser_factory.make(&args); + let mut errors: Vec = vec![]; + let (final_ipv4, final_ipv6, countries) = parser.parse(stdin, &mut errors); + if let Err(error) = generate_rust_code(final_ipv4, final_ipv6, countries, stdout) { + errors.push(format!("Error generating Rust code: {:?}", error)) + } + if errors.is_empty() { + 0 + } else { + let error_list = errors.join("\n"); + write!( + stdout, + r#" + *** DO NOT USE THIS CODE *** + It will produce incorrect results. + The process that generated it found these errors: + +{} + + Fix the errors and regenerate the code. + *** DO NOT USE THIS CODE *** +"#, + error_list + ) + .expect("expected WANRNING output"); + write!(stderr, "{}", error_list).expect("expected error list output"); + 1 + } +} + +pub trait DBIPParserFactory { + fn make(&self, args: &[String]) -> Box; +} + +pub struct DBIPParserFactoryReal {} + +impl DBIPParserFactory for DBIPParserFactoryReal { + fn make(&self, args: &[String]) -> Box { + if args.contains(&"--csv".to_string()) { + Box::new(CSVParser {}) + } else { + Box::new(MMDBParser::new()) + } + } +} + +pub trait DBIPParser: Any { + fn as_any(&self) -> &dyn Any; + + fn parse( + &self, + stdin: &mut dyn io::Read, + errors: &mut Vec, + ) -> (FinalBitQueue, FinalBitQueue, Countries); +} + +pub fn generate_rust_code( + final_ipv4: FinalBitQueue, + final_ipv6: FinalBitQueue, + countries: Countries, + output: &mut dyn io::Write, +) -> Result<(), io::Error> { + write!(output, "\n// GENERATED CODE: REGENERATE, DO NOT MODIFY!\n")?; + generate_country_list(countries, output)?; + generate_country_block_code( + "ipv4_country", + final_ipv4.bit_queue, + output, + final_ipv4.block_count, + )?; + generate_country_block_code( + "ipv6_country", + final_ipv6.bit_queue, + output, + final_ipv6.block_count, + )?; + Ok(()) +} + +fn generate_country_list( + countries: Countries, + output: &mut dyn io::Write, +) -> Result<(), io::Error> { + writeln!(output)?; + writeln!(output, "use lazy_static::lazy_static;")?; + writeln!(output, "use crate::countries::Countries;")?; + writeln!(output)?; + writeln!(output, "lazy_static! {{")?; + writeln!( + output, + " pub static ref COUNTRIES: Countries = Countries::new(" + )?; + writeln!(output, " vec![")?; + for country in countries.iter() { + writeln!( + output, + " (\"{}\", \"{}\"),", + country.iso3166, country.name + )?; + } + writeln!(output, " ]")?; + writeln!(output, " .into_iter()")?; + writeln!( + output, + " .map(|(iso3166, name)| (iso3166.to_string(), name.to_string()))" + )?; + writeln!(output, " .collect::>()")?; + writeln!(output, " );")?; + writeln!(output, "}}")?; + Ok(()) +} + +fn generate_country_block_code( + name: &str, + mut bit_queue: BitQueue, + output: &mut dyn io::Write, + block_count: usize, +) -> Result<(), io::Error> { + let bit_queue_len = bit_queue.len(); + writeln!(output)?; + writeln!(output, "pub fn {}_data() -> (Vec, usize) {{", name)?; + writeln!(output, " (")?; + write!(output, " vec![")?; + let mut values_written = 0usize; + while bit_queue.len() >= COUNTRY_BLOCK_BIT_SIZE { + write_value( + &mut bit_queue, + COUNTRY_BLOCK_BIT_SIZE, + &mut values_written, + output, + )?; + } + if !bit_queue.is_empty() { + let bit_count = bit_queue.len(); + write_value(&mut bit_queue, bit_count, &mut values_written, output)?; + } + write!(output, "\n ],\n")?; + writeln!(output, " {}", bit_queue_len)?; + writeln!(output, " )")?; + writeln!(output, "}}")?; + writeln!(output, "\npub fn {}_block_count() -> usize {{", name)?; + writeln!(output, " {}", block_count)?; + writeln!(output, "}}")?; + Ok(()) +} + +fn write_value( + bit_queue: &mut BitQueue, + bit_count: usize, + values_written: &mut usize, + output: &mut dyn io::Write, +) -> Result<(), io::Error> { + if (*values_written & 0b11) == 0 { + write!(output, "\n ")?; + } else { + write!(output, " ")?; + } + let value = bit_queue + .take_bits(bit_count) + .expect("There should be bits left!"); + write!(output, "0x{:016X},", value)?; + *values_written += 1; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use lazy_static::lazy_static; + use std::any::TypeId; + use std::cell::RefCell; + use std::io::{Error, ErrorKind}; + use std::sync::{Arc, Mutex}; + use test_utilities::byte_array_reader_writer::{ByteArrayReader, ByteArrayWriter}; + + struct DBIPParserMock { + parse_params: Arc>>>, + parse_errors: RefCell>>, + parse_results: RefCell>, + } + + impl DBIPParser for DBIPParserMock { + fn as_any(&self) -> &dyn Any { + self + } + + fn parse( + &self, + _stdin: &mut dyn io::Read, + errors: &mut Vec, + ) -> (FinalBitQueue, FinalBitQueue, Countries) { + self.parse_params.lock().unwrap().push(errors.clone()); + errors.extend(self.parse_errors.borrow_mut().remove(0)); + self.parse_results.borrow_mut().remove(0) + } + } + + impl DBIPParserMock { + pub fn new() -> Self { + Self { + parse_params: Arc::new(Mutex::new(vec![])), + parse_errors: RefCell::new(vec![]), + parse_results: RefCell::new(vec![]), + } + } + + pub fn parse_params(mut self, params: &Arc>>>) -> Self { + self.parse_params = params.clone(); + self + } + + pub fn parse_errors(self, errors: Vec<&str>) -> Self { + self.parse_errors + .borrow_mut() + .push(errors.into_iter().map(|s| s.to_string()).collect()); + self + } + + pub fn parse_result(self, result: (FinalBitQueue, FinalBitQueue, &Countries)) -> Self { + self.parse_results + .borrow_mut() + .push((result.0, result.1, result.2.clone())); + self + } + } + + struct DBIPParserFactoryMock { + make_params: Arc>>>, + make_results: RefCell>, + } + + impl DBIPParserFactory for DBIPParserFactoryMock { + fn make(&self, args: &[String]) -> Box { + self.make_params.lock().unwrap().push(args.to_vec()); + Box::new(self.make_results.borrow_mut().remove(0)) + } + } + + impl DBIPParserFactoryMock { + pub fn new() -> Self { + Self { + make_params: Arc::new(Mutex::new(vec![])), + make_results: RefCell::new(vec![]), + } + } + + fn make_params(mut self, params: &Arc>>>) -> Self { + self.make_params = params.clone(); + self + } + + fn make_result(self, result: DBIPParserMock) -> Self { + self.make_results.borrow_mut().push(result); + self + } + } + + static TEST_DATA: &str = "I represent test data arriving on standard input."; + lazy_static! { + static ref TEST_COUNTRIES: Countries = Countries::new(vec![ + ("FR".to_string(), "France".to_string()), + ("CA".to_string(), "Canada".to_string()), + ]); + } + + #[test] + fn csv_makes_csv() { + let subject = DBIPParserFactoryReal {}; + + let result = subject.make(&vec!["--csv".to_string()]); + + assert_eq!((*result).as_any().type_id(), TypeId::of::()); + } + + #[test] + fn mmdb_makes_mmdb() { + let subject = DBIPParserFactoryReal {}; + + let result = subject.make(&vec!["--mmdb".to_string()]); + + assert_eq!((*result).as_any().type_id(), TypeId::of::()); + } + + #[test] + fn missing_parameter_makes_mmdb() { + let subject = DBIPParserFactoryReal {}; + + let result = subject.make(&vec![]); + + assert_eq!((*result).as_any().type_id(), TypeId::of::()); + } + + #[test] + fn happy_path_test() { + let mut stdin = ByteArrayReader::new(TEST_DATA.as_bytes()); + let mut stdout = ByteArrayWriter::new(); + let mut stderr = ByteArrayWriter::new(); + let parse_params_arc = Arc::new(Mutex::new(vec![])); + let ipv4_result = final_bit_queue(0x1122334455667788, 12); + let ipv6_result = final_bit_queue(0x8877665544332211, 21); + let parser = DBIPParserMock::new() + .parse_params(&parse_params_arc) + .parse_errors(vec![]) + .parse_result((ipv4_result, ipv6_result, &TEST_COUNTRIES)); + let make_params_arc = Arc::new(Mutex::new(vec![])); + let parser_factory = DBIPParserFactoryMock::new() + .make_params(&make_params_arc) + .make_result(parser); + let args = vec![]; + + let result = ip_country( + args.clone(), + &mut stdin, + &mut stdout, + &mut stderr, + &parser_factory, + ); + + assert_eq!(result, 0); + let make_params = make_params_arc.lock().unwrap(); + assert_eq!(*make_params, vec![args.clone()]); + let parse_params = parse_params_arc.lock().unwrap(); + let expected_parse_params: Vec> = vec![vec![]]; + assert_eq!(*parse_params, expected_parse_params); + let stdout_string = String::from_utf8(stdout.get_bytes()).unwrap(); + let stderr_string = String::from_utf8(stderr.get_bytes()).unwrap(); + assert_eq!( + stdout_string, + r#" +// GENERATED CODE: REGENERATE, DO NOT MODIFY! + +use lazy_static::lazy_static; +use crate::countries::Countries; + +lazy_static! { + pub static ref COUNTRIES: Countries = Countries::new( + vec![ + ("ZZ", "Sentinel"), + ("CA", "Canada"), + ("FR", "France"), + ] + .into_iter() + .map(|(iso3166, name)| (iso3166.to_string(), name.to_string())) + .collect::>() + ); +} + +pub fn ipv4_country_data() -> (Vec, usize) { + ( + vec![ + 0x1122334455667788, + ], + 64 + ) +} + +pub fn ipv4_country_block_count() -> usize { + 12 +} + +pub fn ipv6_country_data() -> (Vec, usize) { + ( + vec![ + 0x8877665544332211, + ], + 64 + ) +} + +pub fn ipv6_country_block_count() -> usize { + 21 +} +"# + .to_string() + ); + assert_eq!(stderr_string, "".to_string()); + } + + #[test] + fn sad_path_test() { + let mut stdin = ByteArrayReader::new(TEST_DATA.as_bytes()); + let mut stdout = ByteArrayWriter::new(); + let mut stderr = ByteArrayWriter::new(); + let parse_params_arc = Arc::new(Mutex::new(vec![])); + let ipv4_result = final_bit_queue(0x1122334455667788, 12); + let ipv6_result = final_bit_queue(0x8877665544332211, 21); + let parser = DBIPParserMock::new() + .parse_params(&parse_params_arc) + .parse_errors(vec!["First error", "Second error"]) + .parse_result((ipv4_result, ipv6_result, &TEST_COUNTRIES)); + let make_params_arc = Arc::new(Mutex::new(vec![])); + let parser_factory = DBIPParserFactoryMock::new() + .make_params(&make_params_arc) + .make_result(parser); + let args = vec!["--csv".to_string()]; + + let result = ip_country( + args.clone(), + &mut stdin, + &mut stdout, + &mut stderr, + &parser_factory, + ); + + assert_eq!(result, 1); + let make_params = make_params_arc.lock().unwrap(); + assert_eq!(*make_params, vec![args.clone()]); + let parse_params = parse_params_arc.lock().unwrap(); + let expected_parse_params: Vec> = vec![vec![]]; + assert_eq!(*parse_params, expected_parse_params); + let stdout_string = String::from_utf8(stdout.get_bytes()).unwrap(); + let stderr_string = String::from_utf8(stderr.get_bytes()).unwrap(); + assert_eq!( + stdout_string, + r#" +// GENERATED CODE: REGENERATE, DO NOT MODIFY! + +use lazy_static::lazy_static; +use crate::countries::Countries; + +lazy_static! { + pub static ref COUNTRIES: Countries = Countries::new( + vec![ + ("ZZ", "Sentinel"), + ("CA", "Canada"), + ("FR", "France"), + ] + .into_iter() + .map(|(iso3166, name)| (iso3166.to_string(), name.to_string())) + .collect::>() + ); +} + +pub fn ipv4_country_data() -> (Vec, usize) { + ( + vec![ + 0x1122334455667788, + ], + 64 + ) +} + +pub fn ipv4_country_block_count() -> usize { + 12 +} + +pub fn ipv6_country_data() -> (Vec, usize) { + ( + vec![ + 0x8877665544332211, + ], + 64 + ) +} + +pub fn ipv6_country_block_count() -> usize { + 21 +} + + *** DO NOT USE THIS CODE *** + It will produce incorrect results. + The process that generated it found these errors: + +First error +Second error + + Fix the errors and regenerate the code. + *** DO NOT USE THIS CODE *** +"# + .to_string() + ); + assert_eq!( + stderr_string, + r#"First error +Second error"# + .to_string() + ); + } + + #[test] + fn write_error_from_ip_country() { + let stdin = &mut ByteArrayReader::new(TEST_DATA.as_bytes()); + let stdout = &mut ByteArrayWriter::new(); + let stderr = &mut ByteArrayWriter::new(); + stdout.reject_next_write(Error::new(ErrorKind::WriteZero, "Bad file Descriptor")); + let factory = DBIPParserFactoryReal {}; + + let result = ip_country(vec!["--csv".to_string()], stdin, stdout, stderr, &factory); + + assert_eq!(result, 1); + let stdout_string = String::from_utf8(stdout.get_bytes()).unwrap(); + let stderr_string = String::from_utf8(stderr.get_bytes()).unwrap(); + assert_eq!(stderr_string, "Error generating Rust code: Custom { kind: WriteZero, error: \"Bad file Descriptor\" }"); + assert_eq!(stdout_string, "\n *** DO NOT USE THIS CODE ***\n It will produce incorrect results.\n The process that generated it found these errors:\n\nError generating Rust code: Custom { kind: WriteZero, error: \"Bad file Descriptor\" }\n\n Fix the errors and regenerate the code.\n *** DO NOT USE THIS CODE ***\n"); + } + + fn final_bit_queue(contents: u64, block_count: usize) -> FinalBitQueue { + let mut bit_queue = BitQueue::new(); + bit_queue.add_bits(contents, 64); + FinalBitQueue { + bit_queue, + block_count, + } + } +} diff --git a/ip_country/src/ip_country_csv.rs b/ip_country/src/ip_country_csv.rs new file mode 100644 index 000000000..869361e96 --- /dev/null +++ b/ip_country/src/ip_country_csv.rs @@ -0,0 +1,741 @@ +use crate::countries::Countries; +use crate::country_block_serde::{CountryBlockSerializer, FinalBitQueue}; +use crate::country_block_stream::{CountryBlock, IpRange}; +use crate::ip_country::DBIPParser; +use csv::{StringRecord, StringRecordIter}; +use lazy_static::lazy_static; +use std::any::Any; +use std::fmt::Display; +use std::io; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; +use std::str::FromStr; + +lazy_static! { + static ref HARD_CODED_COUNTRIES: Countries = Countries::new( + vec![ + ("ZZ", "Sentinel"), + ("AD", "Andorra"), + ("AE", "United Arab Emirates"), + ("AF", "Afghanistan"), + ("AG", "Antigua and Barbuda"), + ("AI", "Anguilla"), + ("AL", "Albania"), + ("AM", "Armenia"), + ("AO", "Angola"), + ("AQ", "Antarctica"), + ("AR", "Argentina"), + ("AS", "American Samoa"), + ("AT", "Austria"), + ("AU", "Australia"), + ("AW", "Aruba"), + ("AX", "Aland Islands"), + ("AZ", "Azerbaijan"), + ("BA", "Bosnia and Herzegovina"), + ("BB", "Barbados"), + ("BD", "Bangladesh"), + ("BE", "Belgium"), + ("BF", "Burkina Faso"), + ("BG", "Bulgaria"), + ("BH", "Bahrain"), + ("BI", "Burundi"), + ("BJ", "Benin"), + ("BL", "Saint Barthelemy"), + ("BM", "Bermuda"), + ("BN", "Brunei"), + ("BO", "Bolivia"), + ("BQ", "Bonaire, Saint Eustatius and Saba "), + ("BR", "Brazil"), + ("BS", "Bahamas"), + ("BT", "Bhutan"), + ("BV", "Bouvet Island"), + ("BW", "Botswana"), + ("BY", "Belarus"), + ("BZ", "Belize"), + ("CA", "Canada"), + ("CC", "Cocos Islands"), + ("CD", "Democratic Republic of the Congo"), + ("CF", "Central African Republic"), + ("CG", "Republic of the Congo"), + ("CH", "Switzerland"), + ("CI", "Ivory Coast"), + ("CK", "Cook Islands"), + ("CL", "Chile"), + ("CM", "Cameroon"), + ("CN", "China"), + ("CO", "Colombia"), + ("CR", "Costa Rica"), + ("CU", "Cuba"), + ("CV", "Cabo Verde"), + ("CW", "Curacao"), + ("CX", "Christmas Island"), + ("CY", "Cyprus"), + ("CZ", "Czechia"), + ("DE", "Germany"), + ("DJ", "Djibouti"), + ("DK", "Denmark"), + ("DM", "Dominica"), + ("DO", "Dominican Republic"), + ("DZ", "Algeria"), + ("EC", "Ecuador"), + ("EE", "Estonia"), + ("EG", "Egypt"), + ("EH", "Western Sahara"), + ("ER", "Eritrea"), + ("ES", "Spain"), + ("ET", "Ethiopia"), + ("FI", "Finland"), + ("FJ", "Fiji"), + ("FK", "Falkland Islands"), + ("FM", "Micronesia"), + ("FO", "Faroe Islands"), + ("FR", "France"), + ("GA", "Gabon"), + ("GB", "United Kingdom"), + ("GD", "Grenada"), + ("GE", "Georgia"), + ("GF", "French Guiana"), + ("GG", "Guernsey"), + ("GH", "Ghana"), + ("GI", "Gibraltar"), + ("GL", "Greenland"), + ("GM", "Gambia"), + ("GN", "Guinea"), + ("GP", "Guadeloupe"), + ("GQ", "Equatorial Guinea"), + ("GR", "Greece"), + ("GS", "South Georgia and the South Sandwich Islands"), + ("GT", "Guatemala"), + ("GU", "Guam"), + ("GW", "Guinea-Bissau"), + ("GY", "Guyana"), + ("HK", "Hong Kong"), + ("HM", "Heard Island and McDonald Islands"), + ("HN", "Honduras"), + ("HR", "Croatia"), + ("HT", "Haiti"), + ("HU", "Hungary"), + ("ID", "Indonesia"), + ("IE", "Ireland"), + ("IL", "Israel"), + ("IM", "Isle of Man"), + ("IN", "India"), + ("IO", "British Indian Ocean Territory"), + ("IQ", "Iraq"), + ("IR", "Iran"), + ("IS", "Iceland"), + ("IT", "Italy"), + ("JE", "Jersey"), + ("JM", "Jamaica"), + ("JO", "Jordan"), + ("JP", "Japan"), + ("KE", "Kenya"), + ("KG", "Kyrgyzstan"), + ("KH", "Cambodia"), + ("KI", "Kiribati"), + ("KM", "Comoros"), + ("KN", "Saint Kitts and Nevis"), + ("KP", "North Korea"), + ("KR", "South Korea"), + ("KW", "Kuwait"), + ("KY", "Cayman Islands"), + ("KZ", "Kazakhstan"), + ("LA", "Laos"), + ("LB", "Lebanon"), + ("LC", "Saint Lucia"), + ("LI", "Liechtenstein"), + ("LK", "Sri Lanka"), + ("LR", "Liberia"), + ("LS", "Lesotho"), + ("LT", "Lithuania"), + ("LU", "Luxembourg"), + ("LV", "Latvia"), + ("LY", "Libya"), + ("MA", "Morocco"), + ("MC", "Monaco"), + ("MD", "Moldova"), + ("ME", "Montenegro"), + ("MF", "Saint Martin"), + ("MG", "Madagascar"), + ("MH", "Marshall Islands"), + ("MK", "North Macedonia"), + ("ML", "Mali"), + ("MM", "Myanmar"), + ("MN", "Mongolia"), + ("MO", "Macao"), + ("MP", "Northern Mariana Islands"), + ("MQ", "Martinique"), + ("MR", "Mauritania"), + ("MS", "Montserrat"), + ("MT", "Malta"), + ("MU", "Mauritius"), + ("MV", "Maldives"), + ("MW", "Malawi"), + ("MX", "Mexico"), + ("MY", "Malaysia"), + ("MZ", "Mozambique"), + ("NA", "Namibia"), + ("NC", "New Caledonia"), + ("NE", "Niger"), + ("NF", "Norfolk Island"), + ("NG", "Nigeria"), + ("NI", "Nicaragua"), + ("NL", "The Netherlands"), + ("NO", "Norway"), + ("NP", "Nepal"), + ("NR", "Nauru"), + ("NU", "Niue"), + ("NZ", "New Zealand"), + ("OM", "Oman"), + ("PA", "Panama"), + ("PE", "Peru"), + ("PF", "French Polynesia"), + ("PG", "Papua New Guinea"), + ("PH", "Philippines"), + ("PK", "Pakistan"), + ("PL", "Poland"), + ("PM", "Saint Pierre and Miquelon"), + ("PN", "Pitcairn"), + ("PR", "Puerto Rico"), + ("PS", "Palestinian Territory"), + ("PT", "Portugal"), + ("PW", "Palau"), + ("PY", "Paraguay"), + ("QA", "Qatar"), + ("RE", "Reunion"), + ("RO", "Romania"), + ("RS", "Serbia"), + ("RU", "Russia"), + ("RW", "Rwanda"), + ("SA", "Saudi Arabia"), + ("SB", "Solomon Islands"), + ("SC", "Seychelles"), + ("SD", "Sudan"), + ("SE", "Sweden"), + ("SG", "Singapore"), + ("SH", "Saint Helena"), + ("SI", "Slovenia"), + ("SJ", "Svalbard and Jan Mayen"), + ("SK", "Slovakia"), + ("SL", "Sierra Leone"), + ("SM", "San Marino"), + ("SN", "Senegal"), + ("SO", "Somalia"), + ("SR", "Suriname"), + ("SS", "South Sudan"), + ("ST", "Sao Tome and Principe"), + ("SV", "El Salvador"), + ("SX", "Sint Maarten"), + ("SY", "Syria"), + ("SZ", "Eswatini"), + ("TC", "Turks and Caicos Islands"), + ("TD", "Chad"), + ("TF", "French Southern Territories"), + ("TG", "Togo"), + ("TH", "Thailand"), + ("TJ", "Tajikistan"), + ("TK", "Tokelau"), + ("TL", "Timor Leste"), + ("TM", "Turkmenistan"), + ("TN", "Tunisia"), + ("TO", "Tonga"), + ("TR", "Turkey"), + ("TT", "Trinidad and Tobago"), + ("TV", "Tuvalu"), + ("TW", "Taiwan"), + ("TZ", "Tanzania"), + ("UA", "Ukraine"), + ("UG", "Uganda"), + ("UM", "United States Minor Outlying Islands"), + ("US", "United States"), + ("UY", "Uruguay"), + ("UZ", "Uzbekistan"), + ("VA", "Vatican"), + ("VC", "Saint Vincent and the Grenadines"), + ("VE", "Venezuela"), + ("VG", "British Virgin Islands"), + ("VI", "U.S. Virgin Islands"), + ("VN", "Vietnam"), + ("VU", "Vanuatu"), + ("WF", "Wallis and Futuna"), + ("WS", "Samoa"), + ("XK", "Kosovo"), + ("YE", "Yemen"), + ("YT", "Mayotte"), + ("ZA", "South Africa"), + ("ZM", "Zambia"), + ("ZW", "Zimbabwe"), + ] + .into_iter() + .map(|(iso3166, name)| (iso3166.to_string(), name.to_string())) + .collect::>() + ); +} + +pub struct CSVParser {} + +impl DBIPParser for CSVParser { + fn as_any(&self) -> &dyn Any { + self + } + + fn parse( + &self, + stdin: &mut dyn io::Read, + errors: &mut Vec, + ) -> (FinalBitQueue, FinalBitQueue, Countries) { + let mut csv_rdr = csv::Reader::from_reader(stdin); + let mut serializer = CountryBlockSerializer::new(); + let local_errors = csv_rdr + .records() + .map(|string_record_result| match string_record_result { + Ok(string_record) => { + let countries: &Countries = &HARD_CODED_COUNTRIES; + CountryBlock::try_from((countries, string_record)) + } + Err(e) => Err(format!("CSV format error: {:?}", e)), + }) + .enumerate() + .flat_map(|(idx, country_block_result)| match country_block_result { + Ok(country_block) => { + serializer.add(country_block); + None + } + Err(e) => Some(format!("Line {}: {}", idx + 1, e)), + }) + .collect::>(); + let (final_ipv4, final_ipv6) = serializer.finish(); + errors.extend(local_errors); + (final_ipv4, final_ipv6, HARD_CODED_COUNTRIES.clone()) + } +} + +impl TryFrom<(&Countries, StringRecord)> for CountryBlock { + type Error = String; + + fn try_from( + (countries, string_record): (&Countries, StringRecord), + ) -> Result { + let mut iter = string_record.iter(); + let start_ip = ip_addr_from_iter(&mut iter)?; + let end_ip = ip_addr_from_iter(&mut iter)?; + let iso3166 = match iter.next() { + None => return Err("CSV line contains no ISO 3166 country code".to_string()), + Some(s) => s, + }; + if iter.next().is_some() { + return Err(format!( + "CSV line should contain 3 elements, but contains {}", + string_record.len() + )); + }; + validate_ip_range(start_ip, end_ip)?; + let country = countries.country_from_code(iso3166)?; + let country_block = match (start_ip, end_ip) { + (IpAddr::V4(start), IpAddr::V4(end)) => CountryBlock { + ip_range: IpRange::V4(start, end), + country: country.clone(), + }, + (IpAddr::V6(start), IpAddr::V6(end)) => CountryBlock { + ip_range: IpRange::V6(start, end), + country: country.clone(), + }, + (start, end) => panic!( + "Start and end addresses must be of the same type, not {} and {}", + start, end + ), + }; + Ok(country_block) + } +} + +fn ip_addr_from_iter(iter: &mut StringRecordIter) -> Result { + let ip_string = match iter.next() { + None => return Err("Missing IP address in CSV record".to_string()), + Some(s) => s, + }; + let ip_addr = match IpAddr::from_str(ip_string) { + Err(e) => { + return Err(format!( + "Invalid ({:?}) IP address in CSV record: '{}'", + e, ip_string + )) + } + Ok(ip) => ip, + }; + Ok(ip_addr) +} + +fn validate_ips_are_sequential(start: IP, end: IP) -> Result<(), String> +where + SingleIntegerIPRep: From + PartialOrd, + IP: Display + Copy, +{ + if SingleIntegerIPRep::from(start) > SingleIntegerIPRep::from(end) { + Err(format!( + "Ending address {} is less than starting address {}", + end, start + )) + } else { + Ok(()) + } +} + +fn validate_ip_range(start_ip: IpAddr, end_ip: IpAddr) -> Result<(), String> { + match (start_ip, end_ip) { + (IpAddr::V4(start_v4), IpAddr::V4(end_v4)) => { + validate_ips_are_sequential::(start_v4, end_v4) + } + (IpAddr::V6(start_v6), IpAddr::V6(end_v6)) => { + validate_ips_are_sequential::(start_v6, end_v6) + } + (s, e) => Err(format!( + "Beginning address {} and ending address {} must be the same IP address version", + s, e + )), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::country_block_stream::Country; + use std::cmp::min; + use test_utilities::byte_array_reader_writer::ByteArrayReader; + + static PROPER_TEST_DATA: &str = "0.0.0.0,0.255.255.255,ZZ +1.0.0.0,1.0.0.255,AU +1.0.1.0,1.0.3.255,CN +1.0.4.0,1.0.7.255,AU +1.0.8.0,1.0.15.255,CN +1.0.16.0,1.0.31.255,JP +1.0.32.0,1.0.63.255,CN +1.0.64.0,1.0.127.255,JP +1.0.128.0,1.0.255.255,TH +1.1.0.0,1.1.0.255,CN +0:0:0:0:0:0:0:0,0:255:255:255:0:0:0:0,ZZ +1:0:0:0:0:0:0:0,1:0:0:255:0:0:0:0,AU +1:0:1:0:0:0:0:0,1:0:3:255:0:0:0:0,CN +1:0:4:0:0:0:0:0,1:0:7:255:0:0:0:0,AU +1:0:8:0:0:0:0:0,1:0:15:255:0:0:0:0,CN +1:0:16:0:0:0:0:0,1:0:31:255:0:0:0:0,JP +1:0:32:0:0:0:0:0,1:0:63:255:0:0:0:0,CN +1:0:64:0:0:0:0:0,1:0:127:255:0:0:0:0,JP +1:0:128:0:0:0:0:0,1:0:255:255:0:0:0:0,TH +1:1:0:0:0:0:0:0,1:1:0:255:0:0:0:0,CN +"; + + static BAD_TEST_DATA: &str = "0.0.0.0,0.255.255.255,ZZ +1.0.0.0,1.0.0.255,AU +1.0.1.0,1.0.3.255,CN +1.0.7.255,AU +1.0.8.0,1.0.15.255 +1.0.16.0,1.0.31.255,JP, +BOOGA,BOOGA,BOOGA +1.0.63.255,1.0.32.0,CN +1.0.64.0,1.0.64.0,JP +1.0.128.0,1.0.255.255,TH +1.1.0.0,1.1.0.255,CN +0:0:0:0:0:0:0:0,0:255:255:255:0:0:0:0,ZZ +1:0:0:0:0:0:0:0,1:0:0:255:0:0:0:0,AU +1:0:1:0:0:0:0:0,1:0:3:255:0:0:0:0,CN +1:0:4:0:0:0:0:0,1:0:7:255:0:0:0:0,AU +1:0:8:0:0:0:0:0,1:0:15:255:0:0:0:0,CN +1:0:16:0:0:0:0:0,1:0:31:255:0:0:0:0,JP +BOOGA,BOOGA,BOOGA +1:0:32:0:0:0:0:0,1:0:63:255:0:0:0:0,CN +1:0:64:0:0:0:0:0,1:0:127:255:0:0:0:0,JP +1:0:128:0:0:0:0:0,1:0:255:255:0:0:0:0,TH +1:1:0:0:0:0:0:0,1:1:0:255:0:0:0:0,CN +"; + + #[test] + fn happy_path_test() { + let mut stdin = ByteArrayReader::new(PROPER_TEST_DATA.as_bytes()); + let mut errors = vec![]; + let subject = CSVParser {}; + + let (ipv4_bit_queue, ipv6_bit_queue, countries) = subject.parse(&mut stdin, &mut errors); + + let expected_errors: Vec = vec![]; + assert_eq!(errors, expected_errors); + assert_eq!(countries, HARD_CODED_COUNTRIES.clone()); + assert_eq!(ipv4_bit_queue.bit_queue.len(), 271); + assert_eq!(ipv4_bit_queue.block_count, 11); + let ipv4_compressed: Vec = ipv4_bit_queue.into(); + assert_eq!( + ipv4_compressed, + vec![ + 9259400846767034371, + 153151013337962502, + 5192703286562554892, + 6944551727792783886, + 0 + ] + ); + assert_eq!(ipv6_bit_queue.bit_queue.len(), 1513); + assert_eq!(ipv6_bit_queue.block_count, 20); + let ipv6_compressed: Vec = ipv6_bit_queue.into(); + assert_eq!( + ipv6_compressed, + vec![ + 3458768911871246343, + 54043281427922944, + 12108302188053268224, + 4611686082891046986, + 216173056991952900, + 16161919892486895616, + 422215216529409, + 3075958771080495132, + 432354978795882369, + 13835570455618023424, + 18647717209048234, + 1533581226265280536, + 14483576403638004480, + 2562548218038607874, + 2062837088453, + 30786350749988, + 2112345178780806, + 31525223174541312, + 2163041463893637376, + 13835084483327426560, + 1345171479032233985, + 18014559570755704, + 12433312672202621696, + 122954 + ] + ); + } + + #[test] + fn sad_path_test() { + let mut stdin = ByteArrayReader::new(BAD_TEST_DATA.as_bytes()); + let mut errors = vec![]; + let subject = CSVParser {}; + + let (ipv4_bit_queue, ipv6_bit_queue, countries) = subject.parse(&mut stdin, &mut errors); + + assert_eq!(countries, HARD_CODED_COUNTRIES.clone()); + assert_eq!(ipv4_bit_queue.bit_queue.len(), 239); + assert_eq!(ipv4_bit_queue.block_count, 9); + let ipv4_compressed: Vec = ipv4_bit_queue.into(); + assert_eq!( + ipv4_compressed, + vec![ + 9259400846767034371, + 10385300779421407238, + 12351125828770205212, + 1616904448 + ] + ); + assert_eq!(ipv6_bit_queue.bit_queue.len(), 1513); + assert_eq!(ipv6_bit_queue.block_count, 20); + let ipv6_compressed: Vec = ipv6_bit_queue.into(); + assert_eq!( + ipv6_compressed, + vec![ + 3458768911871246343, + 54043281427922944, + 12108302188053268224, + 4611686082891046986, + 216173056991952900, + 16161919892486895616, + 422215216529409, + 3075958771080495132, + 432354978795882369, + 13835570455618023424, + 18647717209048234, + 1533581226265280536, + 14483576403638004480, + 2562548218038607874, + 2062837088453, + 30786350749988, + 2112345178780806, + 31525223174541312, + 2163041463893637376, + 13835084483327426560, + 1345171479032233985, + 18014559570755704, + 12433312672202621696, + 122954 + ] + ); + assert_eq!(errors, vec![ + "Line 3: CSV format error: Error(UnequalLengths { pos: Some(Position { byte: 67, line: 4, record: 3 }), expected_len: 3, len: 2 })", + "Line 4: CSV format error: Error(UnequalLengths { pos: Some(Position { byte: 80, line: 5, record: 4 }), expected_len: 3, len: 2 })", + "Line 5: CSV format error: Error(UnequalLengths { pos: Some(Position { byte: 99, line: 6, record: 5 }), expected_len: 3, len: 4 })", + "Line 6: Invalid (AddrParseError(Ip)) IP address in CSV record: 'BOOGA'", + "Line 7: Ending address 1.0.32.0 is less than starting address 1.0.63.255", + "Line 17: Invalid (AddrParseError(Ip)) IP address in CSV record: 'BOOGA'", + ]); + } + + fn test_countries() -> Countries { + Countries::old_new(vec![ + Country::new(0, "ZZ", "Sentinel"), + Country::new(1, "AS", "American Samoa"), + Country::new(2, "VN", "Vietnam"), + ]) + } + + #[test] + fn try_from_fails_for_missing_iso3166() { + let string_record = StringRecord::from(vec!["1.2.3.4", "5.6.7.8"]); + + let result = CountryBlock::try_from((&test_countries(), string_record)); + + assert_eq!( + result, + Err("CSV line contains no ISO 3166 country code".to_string()) + ); + } + + #[test] + fn try_from_fails_for_too_many_elements() { + let string_record = StringRecord::from(vec!["1.2.3.4", "5.6.7.8", "US", "extra"]); + + let result = CountryBlock::try_from((&test_countries(), string_record)); + + assert_eq!( + result, + Err("CSV line should contain 3 elements, but contains 4".to_string()) + ); + } + + #[test] + fn try_from_works_for_ipv4() { + let string_record = StringRecord::from(vec!["1.2.3.4", "5.6.7.8", "AS"]); + + let result = CountryBlock::try_from((&test_countries(), string_record)); + + assert_eq!( + result, + Ok(CountryBlock { + ip_range: IpRange::V4( + Ipv4Addr::from_str("1.2.3.4").unwrap(), + Ipv4Addr::from_str("5.6.7.8").unwrap() + ), + country: test_countries().country_from_code("AS").unwrap().clone(), + }) + ); + } + + #[test] + fn try_from_works_for_ipv6() { + let string_record = StringRecord::from(vec![ + "1234:2345:3456:4567:5678:6789:789A:89AB", + "4321:5432:6543:7654:8765:9876:A987:BA98", + "VN", + ]); + + let result = CountryBlock::try_from((&test_countries(), string_record)); + + assert_eq!( + result, + Ok(CountryBlock { + ip_range: IpRange::V6( + Ipv6Addr::from_str("1234:2345:3456:4567:5678:6789:789A:89AB").unwrap(), + Ipv6Addr::from_str("4321:5432:6543:7654:8765:9876:A987:BA98").unwrap() + ), + country: test_countries().country_from_code("VN").unwrap().clone(), + }) + ); + } + + #[test] + fn try_from_fails_for_bad_ip_syntax() { + let string_record = StringRecord::from(vec!["Ooga", "Booga", "AS"]); + + let result = CountryBlock::try_from((&test_countries(), string_record)); + + assert_eq!( + result, + Err("Invalid (AddrParseError(Ip)) IP address in CSV record: 'Ooga'".to_string()) + ); + } + + #[test] + fn try_from_fails_for_missing_start_ip() { + let strings: Vec<&str> = vec![]; + let string_record = StringRecord::from(strings); + + let result = CountryBlock::try_from((&test_countries(), string_record)); + + assert_eq!(result, Err("Missing IP address in CSV record".to_string())); + } + + #[test] + fn try_from_fails_for_missing_end_ip() { + let string_record = StringRecord::from(vec!["1.2.3.4"]); + + let result = CountryBlock::try_from((&test_countries(), string_record)) + .err() + .unwrap(); + + assert_eq!(result, "Missing IP address in CSV record".to_string()); + } + + #[test] + fn try_from_fails_for_reversed_ipv4_addresses() { + let string_record = StringRecord::from(vec!["1.2.3.4", "1.2.3.3", "ZZ"]); + + let result = CountryBlock::try_from((&test_countries(), string_record)); + + assert_eq!( + result, + Err("Ending address 1.2.3.3 is less than starting address 1.2.3.4".to_string()) + ); + } + + #[test] + fn try_from_fails_for_reversed_ipv6_addresses() { + let string_record = StringRecord::from(vec!["1:2:3:4:5:6:7:8", "1:2:3:4:5:6:7:7", "ZZ"]); + + let result = CountryBlock::try_from((&test_countries(), string_record)); + + assert_eq!( + result, + Err( + "Ending address 1:2:3:4:5:6:7:7 is less than starting address 1:2:3:4:5:6:7:8" + .to_string() + ) + ); + } + + #[test] + fn try_from_fails_for_mixed_ip_types() { + let string_record_46 = StringRecord::from(vec!["4.3.2.1", "1:2:3:4:5:6:7:8", "ZZ"]); + let string_record_64 = StringRecord::from(vec!["1:2:3:4:5:6:7:8", "4.3.2.1", "ZZ"]); + + let result_46 = CountryBlock::try_from((&test_countries(), string_record_46)); + let result_64 = CountryBlock::try_from((&test_countries(), string_record_64)); + + assert_eq!(result_46, Err("Beginning address 4.3.2.1 and ending address 1:2:3:4:5:6:7:8 must be the same IP address version".to_string())); + assert_eq!(result_64, Err("Beginning address 1:2:3:4:5:6:7:8 and ending address 4.3.2.1 must be the same IP address version".to_string())); + } + + #[test] + fn try_from_fails_for_unrecognized_iso3166() { + let string_record = StringRecord::from(vec!["1.2.3.4", "5.6.7.8", "XY"]); + + let result = CountryBlock::try_from((&test_countries(), string_record)); + + assert_eq!( + result, + Err("'XY' is not a valid ISO3166 country code".to_string()) + ); + } + + impl Into> for FinalBitQueue { + fn into(mut self) -> Vec { + let mut result = vec![]; + while !self.bit_queue.is_empty() { + let bits = self + .bit_queue + .take_bits(min(64, self.bit_queue.len())) + .unwrap(); + result.push(bits); + } + result + } + } +} diff --git a/ip_country/src/ip_country_mmdb.rs b/ip_country/src/ip_country_mmdb.rs new file mode 100644 index 000000000..2c3d10912 --- /dev/null +++ b/ip_country/src/ip_country_mmdb.rs @@ -0,0 +1,435 @@ +use crate::countries::Countries; +use crate::country_block_serde::{CountryBlockSerializer, FinalBitQueue}; +use crate::country_block_stream::{are_consecutive, Country, CountryBlock, IpRange}; +use crate::ip_country::DBIPParser; +use ipnetwork::{IpNetwork, Ipv6Network}; +use itertools::Itertools; +use maxminddb::geoip2::City; +use maxminddb::{Reader, Within}; +use std::any::Any; +use std::collections::HashSet; +use std::io; +use std::net::Ipv6Addr; + +pub struct MMDBParser {} + +impl Default for MMDBParser { + fn default() -> Self { + Self::new() + } +} + +impl DBIPParser for MMDBParser { + fn as_any(&self) -> &dyn Any { + self + } + + fn parse( + &self, + stdin: &mut dyn io::Read, + errors: &mut Vec, + ) -> (FinalBitQueue, FinalBitQueue, Countries) { + let mut bytes: Vec = vec![]; + match stdin.read_to_end(&mut bytes) { + Ok(_) => {} + Err(e) => { + errors.push(format!("Error reading from stdin: {}", e)); + } + }; + let reader = match Reader::from_source(bytes) { + Ok(r) => r, + Err(e) => { + errors.push(format!("Error opening MaxMind DB: {}", e)); + return ( + FinalBitQueue::default(), + FinalBitQueue::default(), + Countries::new(vec![]), + ); + } + }; + let mut country_pairs: HashSet<(String, String)> = HashSet::new(); + let ip_network = Ipv6Network::new(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 0), 0) + .expect("Ipv6Network stopped working"); + let ip_ranges = match reader.within::(IpNetwork::V6(ip_network)) { + Ok(w) => Self::extract_data(w, &mut country_pairs, errors), + Err(e) => { + errors.push(format!("Error creating within iterator: {}", e)); + vec![] + } + }; + + let country_pairs_vec = country_pairs.into_iter().collect_vec(); + let countries = Countries::new(country_pairs_vec); + + let mut make_country_blocks = |ranges: Vec<(String, IpRange)>| { + ranges + .into_iter() + .map( + |(code, ip_range)| match countries.country_from_code(code.as_str()) { + Ok(country) => CountryBlock { + ip_range, + country: country.clone(), + }, + Err(e) => { + errors.push(format!( + "Error finding country from code {} for IP range {:?}: {}", + code, ip_range, e + )); + CountryBlock { + ip_range, + country: Country::new(0, "ZZ", "Unknown"), + } + } + }, + ) + .collect_vec() + }; + let mut serializer = CountryBlockSerializer::new(); + let country_blocks = make_country_blocks(ip_ranges); + country_blocks + .into_iter() + .for_each(|block| serializer.add(block)); + let (ipv4_bit_queue, ipv6_bit_queue) = serializer.finish(); + + (ipv4_bit_queue, ipv6_bit_queue, countries) + } +} + +impl MMDBParser { + pub fn new() -> Self { + Self {} + } + + fn extract_data<'de>( + within: Within<'de, City<'de>, Vec>, + country_pairs: &mut HashSet<(String, String)>, + errors: &mut Vec, + ) -> Vec<(String, IpRange)> { + let mut coded_ranges: Vec<(String, IpRange)> = vec![]; + let mut add_or_coalesce = |cur_code: &str, cur_range: IpRange| { + let new_range_opt = match coded_ranges.last() { + None => None, + Some((last_code, last_range)) => { + if (last_code == cur_code) + && are_consecutive(last_range.end(), cur_range.start()) + { + Some(IpRange::new(last_range.start(), cur_range.end())) + } else { + None + } + } + }; + if let Some(new_range) = new_range_opt { + // coalesce with last range + let _ = coded_ranges.pop(); + coded_ranges.push((cur_code.to_string(), new_range)); + } else { + // add new range + coded_ranges.push((cur_code.to_string(), cur_range)); + } + }; + within.for_each(|item_result| { + match item_result { + Ok(item) => { + let ip_range = Self::ipn_to_range(item.ip_net); + match item.info.country { + Some(country) => { + match (country.iso_code, country.names.map(|ns| ns.get("en").map(|n| n.to_string()))) { + (Some(code), Some(Some(name))) => { + country_pairs.insert((code.to_string(), name)); + add_or_coalesce(code, ip_range); + } + (Some(code), _) => { + errors.push(format!("Country code {:?} found but no name - using 'Unknown'", code)); + country_pairs.insert((code.to_string(), "Unknown".to_string())); + add_or_coalesce(code, ip_range); + } + (None, Some(Some(name))) => { + errors.push(format!("Country code not found for country: {:?} - using Sentinel", name)); + country_pairs.insert(("ZZ".to_string(), "Sentinel".to_string())); + add_or_coalesce("ZZ", ip_range); + } + (None, _) => { + errors.push(format!("Country code and name not found for range: {:?} - using Sentinel", item.ip_net)); + country_pairs.insert(("ZZ".to_string(), "Sentinel".to_string())); + add_or_coalesce("ZZ", ip_range); + } + } + } + None => { + errors.push(format!("No country information found for range: {:?} - using Sentinel", item.ip_net)); + country_pairs.insert(("ZZ".to_string(), "Sentinel".to_string())); + let ip_range = Self::ipn_to_range(item.ip_net); + add_or_coalesce("ZZ", ip_range); + }, + } + }, + Err(e) => { + errors.push(format!("Error processing item: {}", e)); + } + } + }); + coded_ranges + } + + fn ipn_to_range(ipn: IpNetwork) -> IpRange { + match ipn { + IpNetwork::V4(ipn) => IpRange::V4(ipn.network(), ipn.broadcast()), + IpNetwork::V6(ipn) => IpRange::V6(ipn.network(), ipn.broadcast()), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::country_block_serde::{Ipv4CountryBlockDeserializer, Ipv6CountryBlockDeserializer}; + use crate::country_finder::CountryCodeFinder; + use std::cmp::min; + use std::fs::File; + use std::io::Read; + use std::net::{IpAddr, Ipv4Addr}; + use std::path::PathBuf; + use std::str::FromStr; + + struct BadRead { + delegate: Box, + } + + impl Read for BadRead { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + match self.delegate.read(buf) { + Ok(len) => { + if len == 0 { + Err(io::Error::from(io::ErrorKind::BrokenPipe)) + } else { + Ok(len) + } + } + Err(e) => Err(e), + } + } + } + + #[test] + fn bad_stream() { + /* + 54.36.84.100/22,France,FR + 142.44.196.0/25,India,IN + 142.44.196.128/25,India,IN + 5555:5555:5555:5555:5555:5555:5555:5555/96,Czechia,CZ + */ + let file = PathBuf::from("data/country-scratch-out.mmdb"); + let delegate = File::open(&file).unwrap(); + let mut stdin = BadRead { + delegate: Box::new(delegate), + }; + let subject = MMDBParser::new(); + let mut errors = vec![]; + + let result = subject.parse(&mut stdin, &mut errors); + + assert_eq!( + errors, + vec!["Error reading from stdin: broken pipe".to_string()] + ); + let country_code_finder = CountryCodeFinder::new( + &result.2, + country_data_from_bit_queue(result.0), + country_data_from_bit_queue(result.1), + ); + + let ipv4_country = + country_code_finder.find_country(IpAddr::from_str("54.36.84.100").unwrap()); + assert_eq!( + ipv4_country.map(|c| c.iso3166.clone()), + Some("FR".to_string()) + ); + let ipv4_country = + country_code_finder.find_country(IpAddr::from_str("142.44.196.0").unwrap()); + assert_eq!( + ipv4_country.map(|c| c.iso3166.clone()), + Some("IN".to_string()) + ); + let ipv6_country = country_code_finder + .find_country(IpAddr::from_str("5555:5555:5555:5555:5555:5555:5555:5555").unwrap()); + assert_eq!( + ipv6_country.map(|c| c.iso3166.clone()), + Some("CZ".to_string()) + ); + } + + #[test] + fn improperly_formatted() { + /* + + */ + let file = PathBuf::from("data/improperly-formatted.mmdb"); + let mut stdin = File::open(&file).unwrap(); + let subject = MMDBParser::new(); + let mut errors = vec![]; + + let result = subject.parse(&mut stdin, &mut errors); + + assert_eq!( + errors, + vec!["Error opening MaxMind DB: Invalid database: Could not find MaxMind DB metadata in file.".to_string()] + ); + assert_eq!(result.0.block_count, 0); + assert_eq!(result.1.block_count, 0); + assert_eq!(result.2.len(), 1); // ZZ only + } + + #[test] + fn corrupted() { + /* + + */ + let file = PathBuf::from("data/corrupted.mmdb"); + let mut stdin = File::open(&file).unwrap(); + let subject = MMDBParser::new(); + let mut errors = vec![]; + + let result = subject.parse(&mut stdin, &mut errors); + + assert_eq!( + errors, + vec![ + "Error processing item: Invalid database: the MaxMind DB file's data pointer resolves to an invalid location".to_string(), + "Error processing item: Invalid database: the MaxMind DB file's data pointer resolves to an invalid location".to_string(), + "Error processing item: Invalid database: the MaxMind DB file's data pointer resolves to an invalid location".to_string(), + ] + ); + assert_eq!(result.0.block_count, 3); + assert_eq!(result.1.block_count, 3); + assert_eq!(result.2.len(), 3); + } + + #[test] + fn happy_path() { + /* + 54.36.84.100/22,France,FR + 142.44.196.0/25,India,IN + 142.44.196.128/25,India,IN + 5555:5555:5555:5555:5555:5555:5555:5555/96,Czechia,CZ + */ + let file = PathBuf::from("data/country-scratch-out.mmdb"); + let mut stdin = File::open(&file).unwrap(); + let subject = MMDBParser::new(); + let mut errors = vec![]; + + let (ipv4_bits, ipv6_bits, countries) = subject.parse(&mut stdin, &mut errors); + + let ipv4_data = to_u64s(ipv4_bits); + let ipv4_country_blocks = + Ipv4CountryBlockDeserializer::new(ipv4_data, &countries).collect::>(); + let ipv6_data = to_u64s(ipv6_bits); + let ipv6_country_blocks = + Ipv6CountryBlockDeserializer::new(ipv6_data, &countries).collect::>(); + assert_eq!( + ipv4_country_blocks, + vec![ + CountryBlock { + ip_range: IpRange::V4( + Ipv4Addr::new(0, 0, 0, 0).into(), + Ipv4Addr::new(54, 36, 83, 255).into() + ), + country: Country::new(0, "ZZ", "Sentinel") + }, + CountryBlock { + ip_range: IpRange::V4( + Ipv4Addr::new(54, 36, 84, 0).into(), + Ipv4Addr::new(54, 36, 87, 255).into() + ), + country: Country::new(2, "FR", "France") + }, + CountryBlock { + ip_range: IpRange::V4( + Ipv4Addr::new(54, 36, 88, 0).into(), + Ipv4Addr::new(142, 44, 195, 255).into() + ), + country: Country::new(0, "ZZ", "Sentinel") + }, + CountryBlock { + ip_range: IpRange::V4( + Ipv4Addr::new(142, 44, 196, 0).into(), + Ipv4Addr::new(142, 44, 196, 255).into() + ), + country: Country::new(3, "IN", "India") + }, + CountryBlock { + ip_range: IpRange::V4( + Ipv4Addr::new(142, 44, 197, 0).into(), + Ipv4Addr::new(255, 255, 255, 255).into() + ), + country: Country::new(0, "ZZ", "Sentinel") + }, + ] + ); + assert_eq!( + ipv6_country_blocks, + vec![ + CountryBlock { + ip_range: IpRange::V6( + Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 0).into(), + Ipv6Addr::new( + 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5554, 0xFFFF, 0xFFFF + ) + .into() + ), + country: Country::new(0, "ZZ", "Sentinel") + }, + CountryBlock { + ip_range: IpRange::V6( + Ipv6Addr::new(0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0, 0).into(), + Ipv6Addr::new( + 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0xFFFF, 0xFFFF + ) + .into() + ), + country: Country::new(1, "CZ", "Czechia") + }, + CountryBlock { + ip_range: IpRange::V6( + Ipv6Addr::new(0x5555, 0x5555, 0x5555, 0x5555, 0x5555, 0x5556, 0, 0).into(), + Ipv6Addr::new( + 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF + ) + .into() + ), + country: Country::new(0, "ZZ", "Sentinel") + }, + ] + ) + } + + fn country_data_from_bit_queue(mut bit_queue: FinalBitQueue) -> (Vec, usize) { + let len = bit_queue.bit_queue.len(); + let mut result = vec![]; + loop { + let len = bit_queue.bit_queue.len(); + let next_len = min(64, len); + if next_len == 0 { + break; + } + match bit_queue.bit_queue.take_bits(next_len) { + Some(bits) => result.push(bits), + None => break, + } + } + (result, len) + } + + fn to_u64s(mut final_bit_queue: FinalBitQueue) -> (Vec, usize) { + let mut result = vec![]; + let len = final_bit_queue.bit_queue.len(); + let mut bits_remaining = len; + while bits_remaining > 0 { + let bits_to_take = min(64, bits_remaining); + let bits = final_bit_queue.bit_queue.take_bits(bits_to_take).unwrap(); + result.push(bits); + bits_remaining -= bits_to_take; + } + (result, len) + } +} diff --git a/ip_country/src/lib.rs b/ip_country/src/lib.rs new file mode 100644 index 000000000..47b44b4b3 --- /dev/null +++ b/ip_country/src/lib.rs @@ -0,0 +1,12 @@ +// Copyright (c) 2024, MASQ (https://masq.ai) and/or its affiliates. All rights reserved. + +pub mod bit_queue; +pub mod countries; +pub mod country_block_serde; +pub mod country_block_stream; +pub mod country_finder; +pub mod ip_country; +pub mod ip_country_csv; +pub mod ip_country_mmdb; +#[rustfmt::skip] +pub mod dbip_country; diff --git a/ip_country/src/main.rs b/ip_country/src/main.rs new file mode 100644 index 000000000..f72a39d75 --- /dev/null +++ b/ip_country/src/main.rs @@ -0,0 +1,17 @@ +// Copyright (c) 2024, MASQ (https://masq.ai) and/or its affiliates. All rights reserved. + +use ip_country_lib::ip_country::ip_country; +use ip_country_lib::ip_country::DBIPParserFactoryReal; +use std::env; +use std::io; +use std::process; + +pub fn main() { + process::exit(ip_country( + env::args().collect(), + &mut io::stdin(), + &mut io::stdout(), + &mut io::stderr(), + &DBIPParserFactoryReal {}, + )) +} diff --git a/masq/Cargo.toml b/masq/Cargo.toml index 9a4fca0c1..0a6484895 100644 --- a/masq/Cargo.toml +++ b/masq/Cargo.toml @@ -28,6 +28,7 @@ nix = "0.23.0" [dev-dependencies] atty = "0.2.14" +test_utilities = { path = "../test_utilities" } [lib] name = "masq_cli_lib" diff --git a/masq/src/command_context.rs b/masq/src/command_context.rs index 41308e5bb..e648da218 100644 --- a/masq/src/command_context.rs +++ b/masq/src/command_context.rs @@ -156,12 +156,12 @@ mod tests { use crate::test_utils::mocks::TRANSACT_TIMEOUT_MILLIS_FOR_TESTS; use masq_lib::messages::{FromMessageBody, UiCrashRequest, UiSetupRequest}; use masq_lib::messages::{ToMessageBody, UiShutdownRequest, UiShutdownResponse}; - use masq_lib::test_utils::fake_stream_holder::{ByteArrayReader, ByteArrayWriter}; use masq_lib::test_utils::mock_websockets_server::MockWebSocketsServer; use masq_lib::ui_gateway::MessageBody; use masq_lib::ui_gateway::MessagePath::Conversation; use masq_lib::ui_traffic_converter::{TrafficConversionError, UnmarshalError}; use masq_lib::utils::{find_free_port, running_test}; + use test_utilities::byte_array_reader_writer::{ByteArrayReader, ByteArrayWriter}; #[test] fn constant_has_correct_values() { diff --git a/masq/src/command_factory.rs b/masq/src/command_factory.rs index 21222c454..f443ef4df 100644 --- a/masq/src/command_factory.rs +++ b/masq/src/command_factory.rs @@ -8,6 +8,7 @@ use crate::commands::configuration_command::ConfigurationCommand; use crate::commands::connection_status_command::ConnectionStatusCommand; use crate::commands::crash_command::CrashCommand; use crate::commands::descriptor_command::DescriptorCommand; +use crate::commands::exit_location_command::SetExitLocationCommand; use crate::commands::financials_command::FinancialsCommand; use crate::commands::generate_wallets_command::GenerateWalletsCommand; use crate::commands::recover_wallets_command::RecoverWalletsCommand; @@ -52,6 +53,10 @@ impl CommandFactory for CommandFactoryReal { Err(msg) => return Err(CommandSyntax(msg)), }, "descriptor" => Box::new(DescriptorCommand::new()), + "exit-location" => match SetExitLocationCommand::new(pieces) { + Ok(command) => Box::new(command), + Err(msg) => return Err(CommandSyntax(msg)), + }, "financials" => match FinancialsCommand::new(pieces) { Ok(command) => Box::new(command), Err(msg) => return Err(CommandSyntax(msg)), @@ -103,6 +108,7 @@ impl CommandFactoryReal { mod tests { use super::*; use crate::command_factory::CommandFactoryError::UnrecognizedSubcommand; + use masq_lib::messages::CountryGroups; #[test] fn complains_about_unrecognized_subcommand() { @@ -258,6 +264,34 @@ mod tests { ); } + #[test] + fn factory_produces_exit_location() { + let subject = CommandFactoryReal::new(); + + let command = subject + .make(&[ + "exit-location".to_string(), + "--country-codes".to_string(), + "AS".to_string(), + ]) + .unwrap(); + + assert_eq!( + command + .as_any() + .downcast_ref::() + .unwrap(), + &SetExitLocationCommand { + exit_locations: vec![CountryGroups { + country_codes: vec!["AS".to_string()], + priority: 1 + }], + fallback_routing: false, + show_countries: false, + } + ); + } + #[test] fn complains_about_set_configuration_command_with_no_parameters() { let subject = CommandFactoryReal::new(); diff --git a/masq/src/commands/exit_location_command.rs b/masq/src/commands/exit_location_command.rs new file mode 100644 index 000000000..953df1390 --- /dev/null +++ b/masq/src/commands/exit_location_command.rs @@ -0,0 +1,512 @@ +// Copyright (c) 2025, MASQ (https://masq.ai) and/or its affiliates. All rights reserved. + +use crate::command_context::CommandContext; +use crate::commands::commands_common::CommandError::Payload; +use crate::commands::commands_common::{ + transaction, Command, CommandError, STANDARD_COMMAND_TIMEOUT_MILLIS, +}; +use clap::{App, Arg, ArgGroup, SubCommand}; +use masq_lib::constants::EXIT_COUNTRY_MISSING_COUNTRIES_ERROR; +use masq_lib::exit_locations::ExitLocationSet; +use masq_lib::messages::{CountryGroups, UiSetExitLocationRequest, UiSetExitLocationResponse}; +use masq_lib::shared_schema::common_validators; +use masq_lib::{as_any_ref_in_trait_impl, short_writeln}; +use std::fmt::Debug; + +const EXIT_LOCATION_ABOUT: &str = + "If you activate exit-location preferences, all exit Nodes in countries you don't specify will be prohibited: \n\ + that is, if there is no exit Node available in any of your preferred countries, you'll get an error. However, \ + if you just want to make a suggestion, and you don't mind Nodes in other countries being used if nothing is available \ + in your preferred countries, you can specify --fallback-routing, and you'll get no error unless there are no exit Nodes \ + available anywhere.\n\n\ + Here are some example commands:\n\ + masq> exit-location --show-countries // show all country codes available for exit in Database\n\ + masq> exit-location // disable exit-location preferences\n\ + masq> exit-location --fallback-routing // disable exit-location preferences\n\ + masq> exit-location --country-codes \"CZ,PL|SK\" --fallback-routing \n\t// fallback-routing is ON, \"CZ\" and \"PL\" countries have same priority \"1\", \"SK\" has priority \"2\"\n\ + masq> exit-location --country-codes \"CZ|SK\" \n\t// fallback-routing is OFF, \"CZ\" and \"SK\" countries have different priority\n"; + +const COUNTRY_CODES_HELP: &str = "Establish a set of countries that your Node should try to use for exit Nodes. You should choose from the countries that host the \ + Nodes in your Neighborhood. List the countries in order of preference, separated by vertical pipes (|). If your level of preference \ + for a group of countries is the same, separate those countries by commas (,).\n\ + You can specify country codes as follows:\n\n\ + masq> exit-location --country-codes \"CZ,PL|SK\" \n\t// \"CZ\" and \"PL\" countries have same priority \"1\", \"SK\" has priority \"2\" \n\ + masq> exit-location --country-codes \"CZ|SK\" \n\t// \"CZ\" and \"SK\" countries have different priority\n\n"; + +const FALLBACK_ROUTING_HELP: &str = "If you just want to make a suggestion, and you don't mind Nodes in other countries being used if nothing is available \ + in your preferred countries, you can specify --fallback-routing, and you'll get no error unless there are no exit Nodes \ + available anywhere. \n Here are some examples: \n\n\ + masq> exit-location --country-codes \"CZ\" --fallback-routing \n\t// Set exit-location for \"CZ\" country with fallback-routing on \n\ + masq> exit-location --country-codes \"CZ\" \n\t// Set exit-location for \"CZ\" country with fallback-routing off \n\n"; + +const SHOW_COUNTRIES_HELP: &str = "Use this flag to display all country codes available for exit Nodes in your Neighborhood: \n\n\ + masq> exit-location --show-countries "; + +pub fn exit_location_subcommand() -> App<'static, 'static> { + SubCommand::with_name("exit-location").about(EXIT_LOCATION_ABOUT) +} + +#[derive(Debug, PartialEq, Eq)] +pub struct SetExitLocationCommand { + pub exit_locations: Vec, + pub fallback_routing: bool, + pub show_countries: bool, +} + +impl SetExitLocationCommand { + pub fn new(pieces: &[String]) -> Result { + match set_exit_location_subcommand().get_matches_from_safe(pieces) { + Ok(matches) => { + let exit_locations = match matches.is_present("country-codes") { + true => matches + .values_of("country-codes") + .expect("Expected Country Codes") + .into_iter() + .enumerate() + .map(|(index, code)| CountryGroups::from((code.to_string(), index))) + .collect(), + false => vec![], + }; + let fallback_routing = !matches!( + ( + matches.is_present("fallback-routing"), + matches.is_present("country-codes") + ), + (false, true) + ); + let show_countries = matches.is_present("show-countries"); + Ok(SetExitLocationCommand { + exit_locations, + fallback_routing, + show_countries, + }) + } + + Err(e) => Err(format!("SetExitLocationCommand {}", e)), + } + } +} + +impl Command for SetExitLocationCommand { + fn execute(&self, context: &mut dyn CommandContext) -> Result<(), CommandError> { + let input = UiSetExitLocationRequest { + exit_locations: self.exit_locations.clone(), + fallback_routing: self.fallback_routing, + show_countries: self.show_countries, + }; + let output: Result = + transaction(input, context, STANDARD_COMMAND_TIMEOUT_MILLIS); + match output { + Ok(exit_location_response) => { + if let Some(exit_countries) = exit_location_response.exit_countries { + match !exit_countries.is_empty() { + true => short_writeln!( + context.stdout(), + "Countries available for exit-location: {:?}", + exit_countries + ), + false => short_writeln!( + context.stderr(), + "No countries available for exit-location!" + ), + } + } else { + match exit_location_response.fallback_routing { + true => short_writeln!(context.stdout(), "Fallback Routing is set.",), + false => short_writeln!(context.stdout(), "Fallback Routing NOT set.",), + } + } + if !exit_location_response.exit_country_selection.is_empty() { + let location_set = ExitLocationSet { + locations: exit_location_response.exit_country_selection, + }; + if !exit_location_response.missing_countries.is_empty() { + short_writeln!( + context.stdout(), + "Following countries are missing in Database: {:?}", + exit_location_response.missing_countries + ); + short_writeln!( + context.stderr(), + "code: {}\nmessage: {:?}", + EXIT_COUNTRY_MISSING_COUNTRIES_ERROR, + exit_location_response.missing_countries + ); + } + short_writeln!(context.stdout(), "Exit location set: {}", location_set); + } else if exit_location_response.fallback_routing + && exit_location_response.exit_country_selection.is_empty() + { + short_writeln!(context.stdout(), "Exit location is Unset."); + } + Ok(()) + } + Err(Payload(code, message)) => { + short_writeln!(context.stderr(), "code: {}\nmessage: {}", code, message); + if code == EXIT_COUNTRY_MISSING_COUNTRIES_ERROR { + short_writeln!( + context.stdout(), + "All requested countries are missing in Database: {}", + message + ); + } + Err(Payload(code, message)) + } + Err(err) => { + short_writeln!(context.stderr(), "Error: {}", err); + Err(err) + } + } + } + + as_any_ref_in_trait_impl!(); +} + +pub fn set_exit_location_subcommand() -> App<'static, 'static> { + SubCommand::with_name("exit-location") + .about(EXIT_LOCATION_ABOUT) + .arg( + Arg::with_name("country-codes") + .long("country-codes") + .value_name("COUNTRY-CODES") + .value_delimiter("|") + .validator(common_validators::validate_exit_locations) + .help(COUNTRY_CODES_HELP) + .required(false), + ) + .arg( + Arg::with_name("fallback-routing") + .long("fallback-routing") + .value_name("FALLBACK-ROUTING") + .help(FALLBACK_ROUTING_HELP) + .takes_value(false) + .required(false), + ) + .arg( + Arg::with_name("show-countries") + .long("show-countries") + .value_name("SHOW-COUNTRIES") + .help(SHOW_COUNTRIES_HELP) + .takes_value(false) + .required(false), + ) + .group( + ArgGroup::with_name("show-countries-fallback") + .args(&["show-countries", "fallback-routing"]), + ) + .group( + ArgGroup::with_name("show-countries-codes").args(&["show-countries", "country-codes"]), + ) +} + +#[cfg(test)] +pub mod tests { + use super::*; + use crate::command_context::ContextError; + use crate::command_factory::{CommandFactory, CommandFactoryReal}; + use crate::commands::commands_common::{ + Command, CommandError, STANDARD_COMMAND_TIMEOUT_MILLIS, + }; + use crate::test_utils::mocks::CommandContextMock; + use masq_lib::constants::EXIT_COUNTRY_MISSING_COUNTRIES_ERROR; + use masq_lib::messages::{ + CountryGroups, ExitLocation, ToMessageBody, UiSetExitLocationRequest, + UiSetExitLocationResponse, + }; + use std::sync::{Arc, Mutex}; + + #[test] + fn constants_have_correct_values() { + assert_eq!( + EXIT_LOCATION_ABOUT, + "If you activate exit-location preferences, all exit Nodes in countries you don't specify will be prohibited: \n\ + that is, if there is no exit Node available in any of your preferred countries, you'll get an error. However, \ + if you just want to make a suggestion, and you don't mind Nodes in other countries being used if nothing is available \ + in your preferred countries, you can specify --fallback-routing, and you'll get no error unless there are no exit Nodes \ + available anywhere.\n\n\ + Here are some example commands:\n\ + masq> exit-location --show-countries // show all country codes available for exit in Database\n\ + masq> exit-location // disable exit-location preferences\n\ + masq> exit-location --fallback-routing // disable exit-location preferences\n\ + masq> exit-location --country-codes \"CZ,PL|SK\" --fallback-routing \n\t// fallback-routing is ON, \"CZ\" and \"PL\" countries have same priority \"1\", \"SK\" has priority \"2\"\n\ + masq> exit-location --country-codes \"CZ|SK\" \n\t// fallback-routing is OFF, \"CZ\" and \"SK\" countries have different priority\n" + ); + assert_eq!( + COUNTRY_CODES_HELP, + "Establish a set of countries that your Node should try to use for exit Nodes. You should choose from the countries that host the \ + Nodes in your Neighborhood. List the countries in order of preference, separated by vertical pipes (|). If your level of preference \ + for a group of countries is the same, separate those countries by commas (,).\n\ + You can specify country codes as follows:\n\n\ + masq> exit-location --country-codes \"CZ,PL|SK\" \n\t// \"CZ\" and \"PL\" countries have same priority \"1\", \"SK\" has priority \"2\" \n\ + masq> exit-location --country-codes \"CZ|SK\" \n\t// \"CZ\" and \"SK\" countries have different priority\n\n" + ); + assert_eq!( + FALLBACK_ROUTING_HELP, "If you just want to make a suggestion, and you don't mind Nodes in other countries being used if nothing is available \ + in your preferred countries, you can specify --fallback-routing, and you'll get no error unless there are no exit Nodes \ + available anywhere. \n Here are some examples: \n\n\ + masq> exit-location --country-codes \"CZ\" --fallback-routing \n\t// Set exit-location for \"CZ\" country with fallback-routing on \n\ + masq> exit-location --country-codes \"CZ\" \n\t// Set exit-location for \"CZ\" country with fallback-routing off \n\n" + ); + assert_eq!( + SHOW_COUNTRIES_HELP, + "Use this flag to display all country codes available for exit Nodes in your Neighborhood: \n\n\ + masq> exit-location --show-countries " + ); + } + + #[test] + fn testing_missing_location_error() { + let factory = CommandFactoryReal::new(); + let mut context = + CommandContextMock::new().transact_result(Err(ContextError::PayloadError( + EXIT_COUNTRY_MISSING_COUNTRIES_ERROR, + "\"CZ, SK, IN\"".to_string(), + ))); + let subject = factory.make(&["exit-location".to_string()]).unwrap(); + + let result = subject.execute(&mut context); + let stderr = context.stderr_arc(); + let stdout = context.stdout_arc(); + assert_eq!( + stdout.lock().unwrap().get_string(), + "All requested countries are missing in Database: \"CZ, SK, IN\"\n" + ); + assert_eq!( + stderr.lock().unwrap().get_string(), + "code: 9223372036854775816\nmessage: \"CZ, SK, IN\"\n".to_string() + ); + assert_eq!( + result, + Err(CommandError::Payload( + 9223372036854775816, + "\"CZ, SK, IN\"".to_string() + )) + ); + } + + #[test] + fn testing_handler_for_exit_location_responose() { + let message_body = Ok(UiSetExitLocationResponse { + fallback_routing: true, + exit_country_selection: vec![], + exit_countries: None, + missing_countries: vec![], + } + .tmb(1234)); + + let factory = CommandFactoryReal::new(); + let mut context = CommandContextMock::new().transact_result(message_body); + let subject = factory + .make(&[ + "exit-location".to_string(), + "--fallback-routing".to_string(), + ]) + .unwrap(); + + let result = subject.execute(&mut context); + let stderr = context.stderr_arc(); + let stdout = context.stdout_arc(); + assert_eq!( + stdout.lock().unwrap().get_string(), + "Fallback Routing is set.\nExit location is Unset.\n".to_string() + ); + assert_eq!(stderr.lock().unwrap().get_string(), "".to_string()); + assert_eq!(result, Ok(())); + } + + #[test] + fn can_deserialize_ui_set_exit_location() { + let transact_params_arc = Arc::new(Mutex::new(vec![])); + let mut context = CommandContextMock::new() + .transact_params(&transact_params_arc) + .transact_result(Ok(UiSetExitLocationResponse { + fallback_routing: true, + exit_country_selection: vec![ + ExitLocation { + country_codes: vec!["AO".to_string(), "AS".to_string()], + priority: 1, + }, + ExitLocation { + country_codes: vec!["AD".to_string(), "AS".to_string()], + priority: 2, + }, + ExitLocation { + country_codes: vec!["AS".to_string()], + priority: 3, + }, + ], + exit_countries: None, + missing_countries: vec![], + } + .tmb(0))); + let stderr_arc = context.stderr_arc(); + let stdout_arc = context.stdout_arc(); + let subject = SetExitLocationCommand::new(&[ + "exit-location".to_string(), + "--country-codes".to_string(), + "AO,AS|AD,AS|AS".to_string(), + "--fallback-routing".to_string(), + ]) + .unwrap(); + + let result = subject.execute(&mut context); + + assert_eq!(result, Ok(())); + let expected_request = UiSetExitLocationRequest { + fallback_routing: true, + exit_locations: vec![ + CountryGroups { + country_codes: vec!["AO".to_string(), "AS".to_string()], + priority: 1, + }, + CountryGroups { + country_codes: vec!["AD".to_string(), "AS".to_string()], + priority: 2, + }, + CountryGroups { + country_codes: vec!["AS".to_string()], + priority: 3, + }, + ], + show_countries: false, + }; + let transact_params = transact_params_arc.lock().unwrap(); + let expected_message_body = expected_request.tmb(0); + assert_eq!( + transact_params.as_slice(), + &[(expected_message_body, STANDARD_COMMAND_TIMEOUT_MILLIS)] + ); + let stdout = stdout_arc.lock().unwrap(); + assert_eq!(&stdout.get_string(), "Fallback Routing is set.\nExit location set: Country Codes: [\"AO\", \"AS\"] - Priority: 1; Country Codes: [\"AD\", \"AS\"] - Priority: 2; Country Codes: [\"AS\"] - Priority: 3\n"); + let stderr = stderr_arc.lock().unwrap(); + assert_eq!(&stderr.get_string(), ""); + } + + #[test] + fn absence_of_fallback_routing_produces_fallback_routing_false() { + let transact_params_arc = Arc::new(Mutex::new(vec![])); + let mut context = CommandContextMock::new() + .transact_params(&transact_params_arc) + .transact_result(Ok(UiSetExitLocationResponse { + fallback_routing: false, + exit_country_selection: vec![ExitLocation { + country_codes: vec!["AO".to_string()], + priority: 1, + }], + exit_countries: None, + missing_countries: vec![], + } + .tmb(0))); + let stdout_arc = context.stdout_arc(); + let stderr_arc = context.stderr_arc(); + let subject = SetExitLocationCommand::new(&[ + "exit-location".to_string(), + "--country-codes".to_string(), + "AO".to_string(), + ]) + .unwrap(); + + let result = subject.execute(&mut context); + + assert_eq!(result, Ok(())); + let expected_request = UiSetExitLocationRequest { + fallback_routing: false, + exit_locations: vec![CountryGroups { + country_codes: vec!["AO".to_string()], + priority: 1, + }], + show_countries: false, + }; + let transact_params = transact_params_arc.lock().unwrap(); + let expected_message_body = expected_request.tmb(0); + assert_eq!( + transact_params.as_slice(), + &[(expected_message_body, STANDARD_COMMAND_TIMEOUT_MILLIS)] + ); + let stdout = stdout_arc.lock().unwrap(); + let stderr = stderr_arc.lock().unwrap(); + assert_eq!( + &stdout.get_string(), + "Fallback Routing NOT set.\nExit location set: Country Codes: [\"AO\"] - Priority: 1\n" + ); + assert_eq!(&stderr.get_string(), ""); + } + + #[test] + fn providing_no_arguments_cause_exit_location_reset_request() { + let result = SetExitLocationCommand::new(&["exit-location".to_string()]).unwrap(); + + assert_eq!( + result, + SetExitLocationCommand { + exit_locations: vec![], + fallback_routing: true, + show_countries: false + } + ); + } + + #[test] + fn providing_show_countries_flag_cause_request_for_list_of_countries() { + let transact_params_arc = Arc::new(Mutex::new(vec![])); + let mut context = CommandContextMock::new() + .transact_params(&transact_params_arc) + .transact_result(Ok(UiSetExitLocationResponse { + fallback_routing: false, + exit_country_selection: vec![], + exit_countries: Some(vec!["CZ".to_string()]), + missing_countries: vec![], + } + .tmb(0))); + let stderr_arc = context.stderr_arc(); + let stdout_arc = context.stdout_arc(); + let subject = SetExitLocationCommand::new(&[ + "exit-location".to_string(), + "--show-countries".to_string(), + ]) + .unwrap(); + + let result = subject.execute(&mut context); + + assert_eq!(result, Ok(())); + let expected_request = UiSetExitLocationRequest { + fallback_routing: true, + exit_locations: vec![], + show_countries: true, + }; + let transact_params = transact_params_arc.lock().unwrap(); + let expected_message_body = expected_request.tmb(0); + assert_eq!( + transact_params.as_slice(), + &[(expected_message_body, STANDARD_COMMAND_TIMEOUT_MILLIS)] + ); + let stderr = stderr_arc.lock().unwrap(); + assert_eq!(&stderr.get_string(), ""); + let stdout = stdout_arc.lock().unwrap(); + assert_eq!( + &stdout.get_string(), + "Countries available for exit-location: [\"CZ\"]\n" + ); + } + + #[test] + fn providing_show_countries_with_other_argument_fails() { + let result_expected = + "cannot be used with one or more of the other specified arguments\n\nUSAGE:\n"; + + let result = SetExitLocationCommand::new(&[ + "exit-location".to_string(), + "--show-countries".to_string(), + "--country-codes".to_string(), + "AO".to_string(), + ]) + .unwrap_err(); + + assert!( + result.contains(result_expected), + "result was {:?}, but expected {:?}", + result, + result_expected, + ); + } +} diff --git a/masq/src/commands/mod.rs b/masq/src/commands/mod.rs index 44026ebf8..143dff8e1 100644 --- a/masq/src/commands/mod.rs +++ b/masq/src/commands/mod.rs @@ -7,6 +7,7 @@ pub mod configuration_command; pub mod connection_status_command; pub mod crash_command; pub mod descriptor_command; +pub mod exit_location_command; pub mod financials_command; pub mod generate_wallets_command; pub mod recover_wallets_command; diff --git a/masq/src/interactive_mode.rs b/masq/src/interactive_mode.rs index a9141ffa5..026ed4aef 100644 --- a/masq/src/interactive_mode.rs +++ b/masq/src/interactive_mode.rs @@ -168,10 +168,11 @@ mod tests { CommandFactoryMock, CommandProcessorMock, TerminalActiveMock, TerminalPassiveMock, }; use crossbeam_channel::bounded; - use masq_lib::test_utils::fake_stream_holder::{ByteArrayWriter, FakeStreamHolder}; + use masq_lib::test_utils::fake_stream_holder::FakeStreamHolder; use std::sync::{Arc, Mutex}; use std::thread; use std::time::{Duration, Instant}; + use test_utilities::byte_array_reader_writer::ByteArrayWriter; #[test] fn interactive_mode_works_for_unrecognized_command() { diff --git a/masq/src/notifications/connection_change_notification.rs b/masq/src/notifications/connection_change_notification.rs index 9ea7d5911..0480660fb 100644 --- a/masq/src/notifications/connection_change_notification.rs +++ b/masq/src/notifications/connection_change_notification.rs @@ -34,9 +34,9 @@ impl ConnectionChangeNotification { mod tests { use super::*; use crate::test_utils::mocks::TerminalPassiveMock; - use masq_lib::test_utils::fake_stream_holder::ByteArrayWriter; use masq_lib::utils::running_test; use std::sync::Arc; + use test_utilities::byte_array_reader_writer::ByteArrayWriter; #[test] fn broadcasts_connected_to_neighbor() { diff --git a/masq/src/notifications/crashed_notification.rs b/masq/src/notifications/crashed_notification.rs index 36e0ff4f4..66dfc773c 100644 --- a/masq/src/notifications/crashed_notification.rs +++ b/masq/src/notifications/crashed_notification.rs @@ -63,9 +63,9 @@ impl CrashNotifier { mod tests { use super::*; use crate::test_utils::mocks::TerminalPassiveMock; - use masq_lib::test_utils::fake_stream_holder::ByteArrayWriter; use masq_lib::utils::running_test; use std::sync::Arc; + use test_utilities::byte_array_reader_writer::ByteArrayWriter; #[test] pub fn handles_child_wait_failure() { diff --git a/masq/src/schema.rs b/masq/src/schema.rs index c03a3ea39..287fd9468 100644 --- a/masq/src/schema.rs +++ b/masq/src/schema.rs @@ -8,6 +8,7 @@ use crate::commands::configuration_command::configuration_subcommand; use crate::commands::connection_status_command::connection_status_subcommand; use crate::commands::crash_command::crash_subcommand; use crate::commands::descriptor_command::descriptor_subcommand; +use crate::commands::exit_location_command::exit_location_subcommand; use crate::commands::financials_command::args_validation::financials_subcommand; use crate::commands::generate_wallets_command::generate_wallets_subcommand; use crate::commands::recover_wallets_command::recover_wallets_subcommand; @@ -67,6 +68,7 @@ pub fn app() -> App<'static, 'static> { .subcommand(configuration_subcommand()) .subcommand(connection_status_subcommand()) .subcommand(descriptor_subcommand()) + .subcommand(exit_location_subcommand()) .subcommand(financials_subcommand()) .subcommand(generate_wallets_subcommand()) .subcommand(recover_wallets_subcommand()) diff --git a/masq/src/terminal/integration_test_utils.rs b/masq/src/terminal/integration_test_utils.rs index 7f42f72f5..526e19a66 100644 --- a/masq/src/terminal/integration_test_utils.rs +++ b/masq/src/terminal/integration_test_utils.rs @@ -187,9 +187,9 @@ mod tests { use crate::terminal::terminal_interface::TerminalWrapper; use crate::test_utils::mocks::StdoutBlender; use crossbeam_channel::{bounded, unbounded}; - use masq_lib::test_utils::fake_stream_holder::ByteArrayReader; use std::thread; use std::time::Duration; + use test_utilities::byte_array_reader_writer::ByteArrayReader; #[test] fn constants_have_correct_values() { diff --git a/masq/src/test_utils/mocks.rs b/masq/src/test_utils/mocks.rs index 955fa578a..9f70b7e5b 100644 --- a/masq/src/test_utils/mocks.rs +++ b/masq/src/test_utils/mocks.rs @@ -15,7 +15,6 @@ use linefeed::memory::MemoryTerminal; use linefeed::{Interface, ReadResult, Signal}; use masq_lib::command::StdStreams; use masq_lib::constants::DEFAULT_UI_PORT; -use masq_lib::test_utils::fake_stream_holder::{ByteArrayWriter, ByteArrayWriterInner}; use masq_lib::ui_gateway::MessageBody; use std::cell::RefCell; use std::fmt::Arguments; @@ -23,6 +22,7 @@ use std::io::{Read, Write}; use std::sync::{Arc, Mutex}; use std::time::Duration; use std::{io, thread}; +use test_utilities::byte_array_reader_writer::{ByteArrayWriter, ByteArrayWriterInner}; pub const TRANSACT_TIMEOUT_MILLIS_FOR_TESTS: u64 = DEFAULT_TRANSACT_TIMEOUT_MILLIS; diff --git a/masq_lib/Cargo.toml b/masq_lib/Cargo.toml index af1fd5d15..e845574a8 100644 --- a/masq_lib/Cargo.toml +++ b/masq_lib/Cargo.toml @@ -15,12 +15,14 @@ crossbeam-channel = "0.5.1" dirs = "4.0.0" ethereum-types = "0.9.0" itertools = "0.10.1" +ip_country = { path = "../ip_country"} lazy_static = "1.4.0" log = "0.4.8" regex = "1.5.4" serde = "1.0.133" serde_derive = "1.0.133" serde_json = "1.0.74" +test_utilities = { path = "../test_utilities"} time = {version = "0.3.11", features = [ "formatting" ]} tiny-hderive = "0.3.0" toml = "0.5.8" diff --git a/masq_lib/src/blockchains/blockchain_records.rs b/masq_lib/src/blockchains/blockchain_records.rs index 67a8870e2..cc1198afa 100644 --- a/masq_lib/src/blockchains/blockchain_records.rs +++ b/masq_lib/src/blockchains/blockchain_records.rs @@ -73,16 +73,6 @@ pub struct BlockchainRecord { pub contract_creation_block: u64, } -const POLYGON_MAINNET_CONTRACT_ADDRESS: Address = H160([ - 0xee, 0x9a, 0x35, 0x2f, 0x6a, 0xac, 0x4a, 0xf1, 0xa5, 0xb9, 0xf4, 0x67, 0xf6, 0xa9, 0x3e, 0x0f, - 0xfb, 0xe9, 0xdd, 0x35, -]); - -const ETH_MAINNET_CONTRACT_ADDRESS: Address = H160([ - 0x06, 0xf3, 0xc3, 0x23, 0xf0, 0x23, 0x8c, 0x72, 0xbf, 0x35, 0x01, 0x10, 0x71, 0xf2, 0xb5, 0xb7, - 0xf4, 0x3a, 0x05, 0x4c, -]); - // $tMASQ (Amoy) const POLYGON_AMOY_TESTNET_CONTRACT_ADDRESS: Address = H160([ 0xd9, 0x8c, 0x3e, 0xbd, 0x6b, 0x7f, 0x9b, 0x7c, 0xda, 0x24, 0x49, 0xec, 0xac, 0x00, 0xd1, 0xe5, @@ -110,6 +100,17 @@ const MULTINODE_TESTNET_CONTRACT_ADDRESS: Address = H160([ 0xf1, 0xb3, 0xe6, 0x64, ]); +const ETH_MAINNET_CONTRACT_ADDRESS: Address = H160([ + 0x06, 0xF3, 0xC3, 0x23, 0xf0, 0x23, 0x8c, 0x72, 0xBF, 0x35, 0x01, 0x10, 0x71, 0xf2, 0xb5, 0xB7, + 0xF4, 0x3A, 0x05, 0x4c, +]); + +#[allow(clippy::mixed_case_hex_literals)] +const POLYGON_MAINNET_CONTRACT_ADDRESS: Address = H160([ + 0xEe, 0x9A, 0x35, 0x2F, 0x6a, 0xAc, 0x4a, 0xF1, 0xA5, 0xB9, 0xf4, 0x67, 0xF6, 0xa9, 0x3E, 0x0f, + 0xfB, 0xe9, 0xDd, 0x35, +]); + #[cfg(test)] mod tests { use super::*; diff --git a/masq_lib/src/constants.rs b/masq_lib/src/constants.rs index 155bff5cc..732968ec1 100644 --- a/masq_lib/src/constants.rs +++ b/masq_lib/src/constants.rs @@ -26,6 +26,8 @@ pub const WEIS_IN_GWEI: i128 = 1_000_000_000; pub const DEFAULT_MAX_BLOCK_COUNT: u64 = 100_000; +pub const PAYLOAD_ZERO_SIZE: usize = 0usize; + pub const ETH_MAINNET_CONTRACT_CREATION_BLOCK: u64 = 11_170_708; pub const ETH_ROPSTEN_CONTRACT_CREATION_BLOCK: u64 = 8_688_171; pub const POLYGON_MAINNET_CONTRACT_CREATION_BLOCK: u64 = 14_863_650; @@ -75,6 +77,7 @@ pub const UNMARSHAL_ERROR: u64 = UI_NODE_COMMUNICATION_PREFIX | 4; pub const SETUP_ERROR: u64 = UI_NODE_COMMUNICATION_PREFIX | 5; pub const TIMEOUT_ERROR: u64 = UI_NODE_COMMUNICATION_PREFIX | 6; pub const SCAN_ERROR: u64 = UI_NODE_COMMUNICATION_PREFIX | 7; +pub const EXIT_COUNTRY_MISSING_COUNTRIES_ERROR: u64 = UI_NODE_COMMUNICATION_PREFIX | 8; //accountant pub const ACCOUNTANT_PREFIX: u64 = 0x0040_0000_0000_0000; @@ -104,6 +107,9 @@ pub const BASE_MAINNET_FULL_IDENTIFIER: &str = concatcp!(BASE_FAMILY, LINK, MAIN pub const BASE_SEPOLIA_FULL_IDENTIFIER: &str = concatcp!(BASE_FAMILY, LINK, "sepolia"); pub const DEV_CHAIN_FULL_IDENTIFIER: &str = "dev"; +//allocations +pub const DEFAULT_PREALLOCATION_VEC: usize = 10; + #[cfg(test)] mod tests { use super::*; @@ -198,6 +204,8 @@ mod tests { NODE_RECORD_INNER_CURRENT_VERSION, DataVersion { major: 0, minor: 1 } ); + assert_eq!(PAYLOAD_ZERO_SIZE, 0usize); + assert_eq!(DEFAULT_PREALLOCATION_VEC, 10) } #[test] diff --git a/masq_lib/src/exit_locations.rs b/masq_lib/src/exit_locations.rs new file mode 100644 index 000000000..c5884be54 --- /dev/null +++ b/masq_lib/src/exit_locations.rs @@ -0,0 +1,27 @@ +// Copyright (c) 2025, MASQ (https://masq.ai) and/or its affiliates. All rights reserved. + +use crate::messages::ExitLocation; +use itertools::Itertools; +use std::fmt::{Display, Formatter}; + +pub struct ExitLocationSet { + pub locations: Vec, +} + +impl Display for ExitLocationSet { + fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result { + let exit_location_string = self + .locations + .iter() + .map(|exit_location| { + format!( + "Country Codes: {:?} - Priority: {}", + exit_location.country_codes, exit_location.priority + ) + }) + .collect_vec() + .join("; "); + write!(f, "{}", exit_location_string)?; + Ok(()) + } +} diff --git a/masq_lib/src/lib.rs b/masq_lib/src/lib.rs index e5232b221..1fc5eb68d 100644 --- a/masq_lib/src/lib.rs +++ b/masq_lib/src/lib.rs @@ -1,5 +1,7 @@ // Copyright (c) 2019, MASQ (https://masq.ai) and/or its affiliates. All rights reserved. +extern crate core; + // These must be before the rest of the modules // in order to be able to use the macros. #[macro_use] @@ -20,6 +22,7 @@ pub mod command; pub mod constants; pub mod crash_point; pub mod data_version; +pub mod exit_locations; pub mod shared_schema; pub mod test_utils; pub mod type_obfuscation; diff --git a/masq_lib/src/messages.rs b/masq_lib/src/messages.rs index 45842e419..f8c6effdc 100644 --- a/masq_lib/src/messages.rs +++ b/masq_lib/src/messages.rs @@ -5,6 +5,8 @@ use crate::shared_schema::ConfiguratorError; use crate::ui_gateway::MessageBody; use crate::ui_gateway::MessagePath::{Conversation, FireAndForget}; use crate::utils::to_string; +use core::fmt::Display; +use core::fmt::Formatter; use itertools::Itertools; use serde::de::DeserializeOwned; use serde_derive::{Deserialize, Serialize}; @@ -846,6 +848,69 @@ pub struct UiWalletAddressesResponse { } conversation_message!(UiWalletAddressesResponse, "walletAddresses"); +// CountryGroups are inbound data for ExitLocations from UI. These data structures could be enriched +// in the future according to future user interface needs of more specification +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +pub struct CountryGroups { + #[serde(rename = "CountryGroups")] + pub country_codes: Vec, + pub priority: usize, +} + +impl From<(String, usize)> for CountryGroups { + fn from((country, priority): (String, usize)) -> Self { + CountryGroups { + country_codes: country + .split(',') + .into_iter() + .map(|x| x.to_string()) + .collect::>(), + priority: priority + 1, + } + } +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +pub struct UiSetExitLocationRequest { + #[serde(rename = "fallbackRouting")] + pub fallback_routing: bool, + #[serde(rename = "exitLocations")] + pub exit_locations: Vec, + #[serde(rename = "showCountries")] + pub show_countries: bool, +} +conversation_message!(UiSetExitLocationRequest, "exitLocation"); + +#[derive(Clone, Debug, Eq, Hash, PartialEq, Serialize, Deserialize)] +pub struct ExitLocation { + #[serde(rename = "CountryGroups")] + pub country_codes: Vec, + pub priority: usize, +} + +impl Display for ExitLocation { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "Country Codes: {:?}, Priority: {};", + self.country_codes, self.priority + ) + } +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +pub struct UiSetExitLocationResponse { + #[serde(rename = "fallbackRouting")] + pub fallback_routing: bool, + #[serde(rename = "exitCountrySelection")] + pub exit_country_selection: Vec, + #[serde(rename = "exitCountries")] + pub exit_countries: Option>, + #[serde(rename = "missingCountries")] + pub missing_countries: Vec, +} +conversation_message!(UiSetExitLocationResponse, "exitLocation"); + #[cfg(test)] mod tests { use super::*; diff --git a/masq_lib/src/shared_schema.rs b/masq_lib/src/shared_schema.rs index 276108d2e..11dfb865f 100644 --- a/masq_lib/src/shared_schema.rs +++ b/masq_lib/src/shared_schema.rs @@ -393,7 +393,6 @@ pub fn shared_app(head: App<'static, 'static>) -> App<'static, 'static> { .case_insensitive(true) .hidden(true), ) - .arg(data_directory_arg(DATA_DIRECTORY_HELP)) .arg(db_password_arg(DB_PASSWORD_HELP)) .arg( Arg::with_name("dns-servers") @@ -489,6 +488,7 @@ pub fn shared_app(head: App<'static, 'static>) -> App<'static, 'static> { pub mod common_validators { use crate::constants::LOWEST_USABLE_INSECURE_PORT; + use ip_country_lib::dbip_country::COUNTRIES; use regex::Regex; use std::net::IpAddr; use std::str::FromStr; @@ -525,6 +525,44 @@ pub mod common_validators { } } + pub fn validate_country_code(country_code: &str) -> Result<(), String> { + match COUNTRIES.country_from_code(country_code).is_ok() { + true => Ok(()), + false => Err(format!( + "'{}' is not a valid ISO3166 country code", + country_code + )), + } + } + + pub fn validate_exit_locations(exit_location: String) -> Result<(), String> { + validate_pipe_separated_values(exit_location, |country: String| { + let mut collect_fails = vec![]; + country.split(',').into_iter().for_each(|country_code| { + match validate_country_code(country_code) { + Ok(_) => (), + Err(e) => collect_fails.push(e), + } + }); + match collect_fails.is_empty() { + true => Ok(()), + false => Err(collect_fails.join(", ")), + } + }) + } + + pub fn validate_separate_u64_values(values: String) -> Result<(), String> { + validate_pipe_separated_values(values, |segment: String| { + segment + .parse::() + .map_err(|_| { + "Supply nonnegative numeric values separated by vertical bars like 111|222|333|..." + .to_string() + }) + .map(|_| ()) + }) + } + pub fn validate_private_key(key: String) -> Result<(), String> { if Regex::new("^[0-9a-fA-F]{64}$") .expect("Failed to compile regular expression") @@ -611,16 +649,24 @@ pub mod common_validators { } } - pub fn validate_separate_u64_values(values_with_delimiters: String) -> Result<(), String> { - values_with_delimiters.split('|').try_for_each(|segment| { - segment - .parse::() - .map_err(|_| { - "Supply positive numeric values separated by vertical bars like 111|222|333|..." - .to_string() - }) - .map(|_| ()) - }) + fn validate_pipe_separated_values( + values_with_delimiters: String, + parse_value: fn(String) -> Result<(), String>, + ) -> Result<(), String> { + let mut error_collection = vec![]; + values_with_delimiters + .split('|') + .into_iter() + .for_each(|segment| { + match parse_value(segment.to_string()) { + Ok(_) => (), + Err(msg) => error_collection.push(msg), + }; + }); + match error_collection.is_empty() { + true => Ok(()), + false => Err(error_collection.into_iter().collect::()), + } } } @@ -932,6 +978,23 @@ mod tests { ) } + #[test] + fn validate_exit_key_fails_on_not_valid_country_code() { + let result = common_validators::validate_exit_locations(String::from("AD|AO,RR,XP")); + + assert_eq!( + result, + Err("'RR' is not a valid ISO3166 country code, 'XP' is not a valid ISO3166 country code".to_string()) + ); + } + + #[test] + fn validate_exit_key_success() { + let result = common_validators::validate_exit_locations(String::from("AD|AS")); + + assert_eq!(result, Ok(())); + } + #[test] fn validate_private_key_requires_a_key_that_is_64_characters_long() { let result = common_validators::validate_private_key(String::from("42")); @@ -1082,7 +1145,7 @@ mod tests { assert_eq!( result, Err(String::from( - "Supply positive numeric values separated by vertical bars like 111|222|333|..." + "Supply nonnegative numeric values separated by vertical bars like 111|222|333|..." )) ) } @@ -1094,7 +1157,7 @@ mod tests { assert_eq!( result, Err(String::from( - "Supply positive numeric values separated by vertical bars like 111|222|333|..." + "Supply nonnegative numeric values separated by vertical bars like 111|222|333|..." )) ) } @@ -1106,7 +1169,7 @@ mod tests { assert_eq!( result, Err(String::from( - "Supply positive numeric values separated by vertical bars like 111|222|333|..." + "Supply nonnegative numeric values separated by vertical bars like 111|222|333|..." )) ) } diff --git a/masq_lib/src/test_utils/fake_stream_holder.rs b/masq_lib/src/test_utils/fake_stream_holder.rs index d971ffa7a..d57ed3c3c 100644 --- a/masq_lib/src/test_utils/fake_stream_holder.rs +++ b/masq_lib/src/test_utils/fake_stream_holder.rs @@ -1,136 +1,7 @@ // Copyright (c) 2019, MASQ (https://masq.ai) and/or its affiliates. All rights reserved. use crate::command::StdStreams; -use std::cmp::min; -use std::io; -use std::io::Read; -use std::io::Write; -use std::io::{BufRead, Error}; -use std::sync::{Arc, Mutex}; - -pub struct ByteArrayWriter { - inner_arc: Arc>, -} - -pub struct ByteArrayWriterInner { - byte_array: Vec, - next_error: Option, -} - -impl ByteArrayWriterInner { - pub fn get_bytes(&self) -> Vec { - self.byte_array.clone() - } - pub fn get_string(&self) -> String { - String::from_utf8(self.get_bytes()).unwrap() - } -} - -impl Default for ByteArrayWriter { - fn default() -> Self { - ByteArrayWriter { - inner_arc: Arc::new(Mutex::new(ByteArrayWriterInner { - byte_array: vec![], - next_error: None, - })), - } - } -} - -impl ByteArrayWriter { - pub fn new() -> ByteArrayWriter { - Self::default() - } - - pub fn inner_arc(&self) -> Arc> { - self.inner_arc.clone() - } - - pub fn get_bytes(&self) -> Vec { - self.inner_arc.lock().unwrap().byte_array.clone() - } - pub fn get_string(&self) -> String { - String::from_utf8(self.get_bytes()).unwrap() - } - - pub fn reject_next_write(&mut self, error: Error) { - self.inner_arc().lock().unwrap().next_error = Some(error); - } -} - -impl Write for ByteArrayWriter { - fn write(&mut self, buf: &[u8]) -> io::Result { - let mut inner = self.inner_arc.lock().unwrap(); - if let Some(next_error) = inner.next_error.take() { - Err(next_error) - } else { - for byte in buf { - inner.byte_array.push(*byte) - } - Ok(buf.len()) - } - } - - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } -} - -pub struct ByteArrayReader { - byte_array: Vec, - position: usize, - next_error: Option, -} - -impl ByteArrayReader { - pub fn new(byte_array: &[u8]) -> ByteArrayReader { - ByteArrayReader { - byte_array: byte_array.to_vec(), - position: 0, - next_error: None, - } - } - - pub fn reject_next_read(mut self, error: Error) -> ByteArrayReader { - self.next_error = Some(error); - self - } -} - -impl Read for ByteArrayReader { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - match self.next_error.take() { - Some(error) => Err(error), - None => { - let to_copy = min(buf.len(), self.byte_array.len() - self.position); - #[allow(clippy::needless_range_loop)] - for idx in 0..to_copy { - buf[idx] = self.byte_array[self.position + idx] - } - self.position += to_copy; - Ok(to_copy) - } - } - } -} - -impl BufRead for ByteArrayReader { - fn fill_buf(&mut self) -> io::Result<&[u8]> { - match self.next_error.take() { - Some(error) => Err(error), - None => Ok(&self.byte_array[self.position..]), - } - } - - fn consume(&mut self, amt: usize) { - let result = self.position + amt; - self.position = if result < self.byte_array.len() { - result - } else { - self.byte_array.len() - } - } -} +use test_utilities::byte_array_reader_writer::{ByteArrayReader, ByteArrayWriter}; pub struct FakeStreamHolder { pub stdin: ByteArrayReader, diff --git a/masq_lib/src/test_utils/logging.rs b/masq_lib/src/test_utils/logging.rs index ec60ea72f..92566a753 100644 --- a/masq_lib/src/test_utils/logging.rs +++ b/masq_lib/src/test_utils/logging.rs @@ -1,6 +1,5 @@ // Copyright (c) 2019, MASQ (https://masq.ai) and/or its affiliates. All rights reserved. use crate::logger::real_format_function; -use crate::test_utils::fake_stream_holder::ByteArrayWriter; use crate::test_utils::utils::to_millis; use lazy_static::lazy_static; use log::set_logger; @@ -13,6 +12,7 @@ use std::sync::{Arc, Mutex, MutexGuard}; use std::thread; use std::time::Duration; use std::time::Instant; +use test_utilities::byte_array_reader_writer::ByteArrayWriter; use time::OffsetDateTime; lazy_static! { diff --git a/masq_lib/src/test_utils/utils.rs b/masq_lib/src/test_utils/utils.rs index fa279a4f4..2c73030c3 100644 --- a/masq_lib/src/test_utils/utils.rs +++ b/masq_lib/src/test_utils/utils.rs @@ -42,10 +42,15 @@ pub fn ensure_node_home_directory_does_not_exist(module: &str, name: &str) -> Pa home_dir } +pub fn recreate_data_dir(home_dir: &Path) -> PathBuf { + let _ = fs::remove_dir_all(home_dir); + let _ = fs::create_dir_all(home_dir); + home_dir.to_path_buf() +} + pub fn ensure_node_home_directory_exists(module: &str, name: &str) -> PathBuf { let home_dir = node_home_directory(module, name); - let _ = fs::remove_dir_all(&home_dir); - let _ = fs::create_dir_all(&home_dir); + let _ = recreate_data_dir(&home_dir); home_dir } diff --git a/multinode_integration_tests/Cargo.toml b/multinode_integration_tests/Cargo.toml index 18bb5a16d..775ee0306 100644 --- a/multinode_integration_tests/Cargo.toml +++ b/multinode_integration_tests/Cargo.toml @@ -13,6 +13,7 @@ crossbeam-channel = "0.5.1" ethereum-types = "0.9.0" ethsign-crypto = "0.2.1" futures = "0.1.31" +ip_country = { path = "../ip_country"} itertools = "0.10.1" lazy_static = "1.4.0" log = "0.4.14" diff --git a/multinode_integration_tests/src/masq_mock_node.rs b/multinode_integration_tests/src/masq_mock_node.rs index f4598be4e..384209e20 100644 --- a/multinode_integration_tests/src/masq_mock_node.rs +++ b/multinode_integration_tests/src/masq_mock_node.rs @@ -147,6 +147,10 @@ impl MASQNode for MASQMockNode { fn routes_data(&self) -> bool { true // just a guess } + + fn country_code_opt(&self) -> Option { + None + } } pub struct MutableMASQMockNode { @@ -400,18 +404,20 @@ impl MASQMockNode { &self, message_type_lite: MessageTypeLite, immediate_neighbor: SocketAddr, + exit_node_cryptde: Option, ) -> Option> { let public_key = self.main_public_key(); let cryptde = CryptDENull::from(public_key, TEST_DEFAULT_MULTINODE_CHAIN); + let exit_cryptde = exit_node_cryptde.unwrap_or_else(|| cryptde.clone()); loop { if let Ok((_, _, live_cores_package)) = self.wait_for_package(&JsonMasquerader::new(), Duration::from_secs(2)) { let (_, intended_exit_public_key) = CryptDENull::extract_key_pair(public_key.len(), &live_cores_package.payload); - assert_eq!(&intended_exit_public_key, public_key); + assert_eq!(&intended_exit_public_key, exit_cryptde.public_key()); let expired_cores_package = live_cores_package - .to_expired(immediate_neighbor, &cryptde, &cryptde) + .to_expired(immediate_neighbor, &cryptde, &exit_cryptde) .unwrap(); if message_type_lite == expired_cores_package.payload.clone().into() { return Some(expired_cores_package); diff --git a/multinode_integration_tests/src/masq_node.rs b/multinode_integration_tests/src/masq_node.rs index 1e2528ed8..20bc4fb49 100644 --- a/multinode_integration_tests/src/masq_node.rs +++ b/multinode_integration_tests/src/masq_node.rs @@ -7,6 +7,7 @@ use masq_lib::constants::{ MASQ_URL_PREFIX, }; use masq_lib::utils::to_string; +use node_lib::neighborhood::node_location::get_node_location; use node_lib::sub_lib::cryptde::{CryptDE, PublicKey}; use node_lib::sub_lib::cryptde_null::CryptDENull; use node_lib::sub_lib::neighborhood::{NodeDescriptor, RatePack}; @@ -210,6 +211,7 @@ pub trait MASQNode: Any { fn chain(&self) -> Chain; fn accepts_connections(&self) -> bool; fn routes_data(&self) -> bool; + fn country_code_opt(&self) -> Option; } pub struct MASQNodeUtils {} @@ -300,6 +302,15 @@ impl MASQNodeUtils { Self::start_from(start.parent().unwrap()) } } + + pub fn derive_country_code_opt(node_addr: &NodeAddr) -> Option { + let country_code = get_node_location(Some(node_addr.ip_addr())); + if let Some(cc) = country_code { + Some(cc.country_code) + } else { + None + } + } } #[cfg(test)] diff --git a/multinode_integration_tests/src/masq_real_node.rs b/multinode_integration_tests/src/masq_real_node.rs index d9dc2c6da..4c1154525 100644 --- a/multinode_integration_tests/src/masq_real_node.rs +++ b/multinode_integration_tests/src/masq_real_node.rs @@ -760,6 +760,10 @@ impl MASQNode for MASQRealNode { fn routes_data(&self) -> bool { self.guts.routes_data } + + fn country_code_opt(&self) -> Option { + MASQNodeUtils::derive_country_code_opt(&self.node_addr()) + } } impl MASQRealNode { diff --git a/multinode_integration_tests/src/multinode_gossip.rs b/multinode_integration_tests/src/multinode_gossip.rs index 5c1f99321..8b50ce2fb 100644 --- a/multinode_integration_tests/src/multinode_gossip.rs +++ b/multinode_integration_tests/src/multinode_gossip.rs @@ -2,13 +2,12 @@ use crate::masq_node::MASQNode; use masq_lib::blockchains::chains::Chain; use masq_lib::test_utils::utils::TEST_DEFAULT_MULTINODE_CHAIN; -use node_lib::neighborhood::gossip::{GossipNodeRecord, Gossip_0v1}; -use node_lib::neighborhood::AccessibleGossipRecord; +use node_lib::neighborhood::gossip::{AccessibleGossipRecord, GossipNodeRecord, Gossip_0v1}; use node_lib::sub_lib::cryptde::PublicKey; use node_lib::sub_lib::cryptde_null::CryptDENull; use node_lib::test_utils::vec_to_set; use std::collections::HashSet; -use std::convert::{TryFrom, TryInto}; +use std::convert::TryInto; use std::net::IpAddr; #[derive(PartialEq, Eq, Clone, Debug)] diff --git a/multinode_integration_tests/src/neighborhood_constructor.rs b/multinode_integration_tests/src/neighborhood_constructor.rs index 3ab6158f5..d5452f201 100644 --- a/multinode_integration_tests/src/neighborhood_constructor.rs +++ b/multinode_integration_tests/src/neighborhood_constructor.rs @@ -6,11 +6,10 @@ use crate::masq_node_cluster::MASQNodeCluster; use crate::masq_real_node::{make_consuming_wallet_info, NodeStartupConfigBuilder}; use crate::masq_real_node::{MASQRealNode, NodeStartupConfig}; use crate::multinode_gossip::{Standard, StandardBuilder}; -use node_lib::neighborhood::gossip::Gossip_0v1; +use node_lib::neighborhood::gossip::{AccessibleGossipRecord, Gossip_0v1}; use node_lib::neighborhood::gossip_producer::{GossipProducer, GossipProducerReal}; use node_lib::neighborhood::neighborhood_database::NeighborhoodDatabase; use node_lib::neighborhood::node_record::{NodeRecord, NodeRecordMetadata}; -use node_lib::neighborhood::AccessibleGossipRecord; use node_lib::sub_lib::cryptde::PublicKey; use node_lib::sub_lib::utils::time_t_timestamp; use node_lib::test_utils::neighborhood_test_utils::db_from_node; @@ -259,6 +258,8 @@ fn from_masq_node_to_node_record(masq_node: &dyn MASQNode) -> NodeRecord { last_update: time_t_timestamp(), node_addr_opt: agr.node_addr_opt.clone(), unreachable_hosts: Default::default(), + node_location_opt: None, + country_undesirability: 0u32, }, signed_gossip: agr.signed_gossip.clone(), signature: agr.signature, diff --git a/multinode_integration_tests/src/utils.rs b/multinode_integration_tests/src/utils.rs index e04b94149..6afbc4d62 100644 --- a/multinode_integration_tests/src/utils.rs +++ b/multinode_integration_tests/src/utils.rs @@ -12,8 +12,8 @@ use node_lib::database::db_initializer::{ }; use node_lib::database::rusqlite_wrappers::ConnectionWrapper; use node_lib::db_config::config_dao::{ConfigDao, ConfigDaoReal}; +use node_lib::neighborhood::gossip::AccessibleGossipRecord; use node_lib::neighborhood::node_record::NodeRecordInner_0v1; -use node_lib::neighborhood::AccessibleGossipRecord; use node_lib::sub_lib::cryptde::{CryptData, PlainData}; use std::collections::BTreeSet; use std::io::{ErrorKind, Read, Write}; @@ -134,6 +134,7 @@ impl From<&dyn MASQNode> for AccessibleGossipRecord { accepts_connections: masq_node.accepts_connections(), routes_data: masq_node.routes_data(), version: 0, + country_code_opt: masq_node.country_code_opt(), }, node_addr_opt: Some(masq_node.node_addr()), signed_gossip: PlainData::new(b""), diff --git a/multinode_integration_tests/tests/bookkeeping_test.rs b/multinode_integration_tests/tests/bookkeeping_test.rs index cbed01e84..6c7552eae 100644 --- a/multinode_integration_tests/tests/bookkeeping_test.rs +++ b/multinode_integration_tests/tests/bookkeeping_test.rs @@ -24,6 +24,7 @@ fn provided_and_consumed_services_are_recorded_in_databases() { .map(|_| start_real_node(&mut cluster, originating_node.node_reference())) .collect::>(); + //TODO card #803 Create function wait_for_gossip thread::sleep(Duration::from_secs(10)); let mut client = originating_node.make_client(8080, STANDARD_CLIENT_TIMEOUT_MILLIS); diff --git a/multinode_integration_tests/tests/communication_failure_test.rs b/multinode_integration_tests/tests/communication_failure_test.rs index 1c96ad483..65d31dd4f 100644 --- a/multinode_integration_tests/tests/communication_failure_test.rs +++ b/multinode_integration_tests/tests/communication_failure_test.rs @@ -12,9 +12,9 @@ use multinode_integration_tests_lib::masq_real_node::{ }; use multinode_integration_tests_lib::neighborhood_constructor::construct_neighborhood; use node_lib::json_masquerader::JsonMasquerader; +use node_lib::neighborhood::gossip::AccessibleGossipRecord; use node_lib::neighborhood::neighborhood_database::NeighborhoodDatabase; use node_lib::neighborhood::node_record::NodeRecord; -use node_lib::neighborhood::AccessibleGossipRecord; use node_lib::sub_lib::cryptde::{CryptDE, PublicKey}; use node_lib::sub_lib::cryptde_null::CryptDENull; use node_lib::sub_lib::hopper::{ @@ -177,6 +177,7 @@ fn dns_resolution_failure_first_automatic_retry_succeeds() { .wait_for_specific_package( MessageTypeLite::ClientRequest, originating_node.socket_addr(PortSelector::First), + None, ) .unwrap(); let dns_fail_pkg = make_package_for_client( @@ -199,6 +200,7 @@ fn dns_resolution_failure_first_automatic_retry_succeeds() { .wait_for_specific_package( MessageTypeLite::ClientRequest, originating_node.socket_addr(PortSelector::First), + None, ) .unwrap(); let sequenced_packet = SequencedPacket::new(EXAMPLE_HTML_RESPONSE.as_bytes().to_vec(), 0, true); @@ -366,6 +368,7 @@ fn dns_resolution_failure_no_longer_blacklists_exit_node_for_all_hosts() { .wait_for_specific_package( MessageTypeLite::ClientRequest, originating_node_socket_address, + None, ) .unwrap(); let dns_fail_pkg = make_package_for_client( @@ -408,6 +411,7 @@ fn dns_resolution_failure_no_longer_blacklists_exit_node_for_all_hosts() { .wait_for_specific_package( MessageTypeLite::ClientRequest, originating_node_socket_address, + None, ) .unwrap(); assert_eq!( diff --git a/multinode_integration_tests/tests/country_code_exit_location_routing.rs b/multinode_integration_tests/tests/country_code_exit_location_routing.rs new file mode 100644 index 000000000..79b043d52 --- /dev/null +++ b/multinode_integration_tests/tests/country_code_exit_location_routing.rs @@ -0,0 +1,106 @@ +// Copyright (c) 2019, MASQ (https://masq.ai) and/or its affiliates. All rights reserved. + +use std::thread; +use std::time::Duration; + +use masq_lib::messages::{ + CountryGroups, ToMessageBody, UiSetConfigurationRequest, UiSetExitLocationRequest, +}; +use masq_lib::test_utils::utils::TEST_DEFAULT_MULTINODE_CHAIN; +use multinode_integration_tests_lib::masq_node::{MASQNode, PortSelector}; +use multinode_integration_tests_lib::masq_node_cluster::MASQNodeCluster; +use multinode_integration_tests_lib::neighborhood_constructor::construct_neighborhood; +use node_lib::sub_lib::cryptde_null::CryptDENull; +use node_lib::sub_lib::hopper::MessageTypeLite; +use node_lib::sub_lib::neighborhood::RatePack; +use node_lib::test_utils::neighborhood_test_utils::{db_from_node, make_node_record}; + +#[test] +fn http_end_to_end_routing_test_with_exit_location() { + let mut cluster = MASQNodeCluster::start().unwrap(); + let first_neighbor = make_node_record(2345, true); + let mut exit_fr = make_node_record(3456, false); + exit_fr.inner.country_code_opt = Some("FR".to_string()); + exit_fr.inner.rate_pack = RatePack { + routing_byte_rate: 100, + routing_service_rate: 100, + exit_byte_rate: 100, + exit_service_rate: 100, + }; + let mut exit_de = make_node_record(4567, false); + exit_de.inner.country_code_opt = Some("DE".to_string()); + exit_de.inner.rate_pack = RatePack { + routing_byte_rate: 1, + routing_service_rate: 1, + exit_byte_rate: 1, + exit_service_rate: 1, + }; + exit_fr.resign(); + exit_de.resign(); + + let dest_db = { + let subject_node_record = make_node_record(1234, true); + let mut dest_db = db_from_node(&subject_node_record); + dest_db.add_node(first_neighbor.clone()).unwrap(); + dest_db.add_arbitrary_full_neighbor( + subject_node_record.public_key(), + first_neighbor.public_key(), + ); + dest_db.add_node(exit_fr.clone()).unwrap(); + dest_db.add_arbitrary_full_neighbor(first_neighbor.public_key(), exit_fr.public_key()); + dest_db.add_node(exit_de.clone()).unwrap(); + dest_db.add_arbitrary_full_neighbor(first_neighbor.public_key(), exit_de.public_key()); + dest_db + }; + + let (_, subject_real_node, mut mock_nodes) = + construct_neighborhood(&mut cluster, dest_db, vec![], |config_builder| { + config_builder.ui_port(51883).build() + }); + + thread::sleep(Duration::from_millis(500 * 6u64)); + + let ui = subject_real_node.make_ui(51883); + ui.send_request( + UiSetConfigurationRequest { + name: "min-hops".to_string(), + value: "2".to_string(), + } + .tmb(0), + ); + ui.send_request( + UiSetExitLocationRequest { + fallback_routing: false, + exit_locations: vec![CountryGroups { + country_codes: vec!["FR".to_string()], + priority: 1, + }], + show_countries: false, + } + .tmb(1), + ); + thread::sleep(Duration::from_millis(500)); + let mut client = subject_real_node.make_client(8080, 5000); + client.send_chunk(b"GET /ip HTTP/1.1\r\nHost: httpbin.org\r\n\r\n"); + + let neighbor_mock = mock_nodes.remove(first_neighbor.public_key()).unwrap(); + let mut expired_cores_package = neighbor_mock + .wait_for_specific_package( + MessageTypeLite::ClientRequest, + subject_real_node.socket_addr(PortSelector::First), + Some(CryptDENull::from( + exit_fr.public_key(), + TEST_DEFAULT_MULTINODE_CHAIN, + )), + ) + .unwrap(); + + let last_hop = expired_cores_package + .remaining_route + .shift(&CryptDENull::from( + neighbor_mock.main_public_key(), + TEST_DEFAULT_MULTINODE_CHAIN, + )) + .unwrap(); + assert_eq!(last_hop.public_key, exit_fr.inner.public_key) +} diff --git a/multinode_integration_tests/tests/data_routing_test.rs b/multinode_integration_tests/tests/data_routing_test.rs index 0c4cef279..cdefcd354 100644 --- a/multinode_integration_tests/tests/data_routing_test.rs +++ b/multinode_integration_tests/tests/data_routing_test.rs @@ -316,7 +316,7 @@ fn multiple_stream_zero_hop_test() { let mut another_client = zero_hop_node.make_client(8080, STANDARD_CLIENT_TIMEOUT_MILLIS); one_client.send_chunk(b"GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n"); - another_client.send_chunk(b"GET /online/ HTTP/1.1\r\nAccept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7\r\nAccept-Language: cs-CZ,cs;q=0.9,en;q=0.8,sk;q=0.7\r\nCache-Control: max-age=0\r\nConnection: keep-alive\r\nHost: whatever.neverssl.com\r\nUpgrade-Insecure-Requests: 1\r\nUser-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36\r\n\r\n"); + another_client.send_chunk(b"GET / HTTP/1.1\r\nAccept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7\r\nAccept-Language: cs-CZ,cs;q=0.9,en;q=0.8,sk;q=0.7\r\nCache-Control: max-age=0\r\nConnection: keep-alive\r\nHost: www.testingmcafeesites.com\r\nUpgrade-Insecure-Requests: 1\r\nUser-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36\r\n\r\n"); let one_response = one_client.wait_for_chunk(); let another_response = another_client.wait_for_chunk(); @@ -330,7 +330,8 @@ fn multiple_stream_zero_hop_test() { assert_eq!( index_of( &another_response, - &b"neverssl.com will never use SSL (also known as TLS)"[..], + &b"This is an index url which gives an overview of the different test urls available." + [..], ) .is_some(), true, diff --git a/multinode_integration_tests/tests/neighbor_selection_test.rs b/multinode_integration_tests/tests/neighbor_selection_test.rs index e40b030b7..225713ba5 100644 --- a/multinode_integration_tests/tests/neighbor_selection_test.rs +++ b/multinode_integration_tests/tests/neighbor_selection_test.rs @@ -9,10 +9,10 @@ use multinode_integration_tests_lib::multinode_gossip::{ use multinode_integration_tests_lib::neighborhood_constructor::{ construct_neighborhood, do_not_modify_config, }; +use node_lib::neighborhood::gossip::AccessibleGossipRecord; use node_lib::neighborhood::gossip::GossipBuilder; use node_lib::neighborhood::neighborhood_database::NeighborhoodDatabase; use node_lib::neighborhood::node_record::NodeRecord; -use node_lib::neighborhood::AccessibleGossipRecord; use node_lib::sub_lib::cryptde::PublicKey; use node_lib::sub_lib::neighborhood::GossipFailure_0v1; use node_lib::test_utils::neighborhood_test_utils::{db_from_node, make_node_record}; @@ -146,7 +146,7 @@ fn node_remembers_its_neighbors_across_a_bounce() { let mut config = originating_node.get_startup_config(); config.neighbors = vec![]; originating_node.restart_node(config); - let (gossip, ip_addr) = relay1.wait_for_gossip(Duration::from_millis(2000)).unwrap(); + let (gossip, ip_addr) = relay1.wait_for_gossip(Duration::from_millis(5000)).unwrap(); match parse_gossip(&gossip, ip_addr) { GossipType::DebutGossip(_) => (), gt => panic!("Expected GossipType::Debut, but found {:?}", gt), diff --git a/node/.idea/modules.xml b/node/.idea/modules.xml index 11464082f..8ce2c17bd 100644 --- a/node/.idea/modules.xml +++ b/node/.idea/modules.xml @@ -8,6 +8,7 @@ + - \ No newline at end of file + diff --git a/node/Cargo.lock b/node/Cargo.lock index 63e2b2029..61b4ef2a8 100644 --- a/node/Cargo.lock +++ b/node/Cargo.lock @@ -430,8 +430,8 @@ version = "0.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef196d5d972878a48da7decb7686eded338b4858fbabeed513d63a7c98b2b82d" dependencies = [ - "proc-macro2 1.0.59", - "quote 1.0.28", + "proc-macro2 1.0.94", + "quote 1.0.40", "unicode-xid 0.2.1", ] @@ -634,6 +634,27 @@ dependencies = [ "subtle 2.4.1", ] +[[package]] +name = "csv" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" +dependencies = [ + "csv-core", + "itoa 1.0.4", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" +dependencies = [ + "memchr", +] + [[package]] name = "ctr" version = "0.3.2" @@ -689,8 +710,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40eebddd2156ce1bb37b20bbe5151340a31828b1f2d22ba4141f3531710e38df" dependencies = [ "convert_case", - "proc-macro2 1.0.59", - "quote 1.0.28", + "proc-macro2 1.0.94", + "quote 1.0.40", "rustc_version 0.3.3", "syn 1.0.85", ] @@ -880,8 +901,8 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" dependencies = [ - "proc-macro2 1.0.59", - "quote 1.0.28", + "proc-macro2 1.0.94", + "quote 1.0.40", "syn 1.0.85", "synstructure", ] @@ -962,7 +983,7 @@ dependencies = [ "lazy_static", "log 0.4.18", "regex", - "thiserror", + "thiserror 1.0.30", "yansi", ] @@ -978,7 +999,7 @@ dependencies = [ "lazy_static", "log 0.4.18", "regex", - "thiserror", + "thiserror 1.0.30", "yansi", ] @@ -1537,6 +1558,17 @@ dependencies = [ "libc", ] +[[package]] +name = "ip_country" +version = "0.1.0" +dependencies = [ + "csv", + "ipnetwork", + "itertools 0.13.0", + "lazy_static", + "maxminddb", +] + [[package]] name = "ipconfig" version = "0.1.9" @@ -1568,6 +1600,12 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68f2d64f2edebec4ce84ad108148e67e1064789bee435edc5b60ad398714a3a9" +[[package]] +name = "ipnetwork" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf370abdafd54d13e54a620e8c3e1145f28e46cc9d704bc6d94414559df41763" + [[package]] name = "itertools" version = "0.8.2" @@ -1586,6 +1624,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "0.4.6" @@ -1644,9 +1691,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.145" +version = "0.2.171" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc86cde3ff845662b8f4ef6cb50ea0e20c524eb3d29ae048287e06a1b3fa6a81" +checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" [[package]] name = "libsecp256k1" @@ -1815,6 +1862,7 @@ dependencies = [ "nix 0.23.1", "num", "regex", + "test_utilities", "thousands", "time 0.3.11", "websocket", @@ -1830,6 +1878,7 @@ dependencies = [ "crossbeam-channel 0.5.1", "dirs 4.0.0", "ethereum-types", + "ip_country", "itertools 0.10.3", "lazy_static", "log 0.4.18", @@ -1838,6 +1887,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", + "test_utilities", "time 0.3.11", "tiny-hderive", "toml", @@ -1856,6 +1906,19 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" +[[package]] +name = "maxminddb" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a197e44322788858682406c74b0b59bf8d9b4954fe1f224d9a25147f1880bba" +dependencies = [ + "ipnetwork", + "log 0.4.18", + "memchr", + "serde", + "thiserror 2.0.12", +] + [[package]] name = "maybe-uninit" version = "2.0.0" @@ -2006,6 +2069,7 @@ dependencies = [ "ethereum-types", "ethsign-crypto", "futures", + "ip_country", "itertools 0.10.3", "lazy_static", "log 0.4.18", @@ -2114,6 +2178,7 @@ dependencies = [ "heck", "http 0.2.5", "indoc", + "ip_country", "ipconfig 0.2.2", "itertools 0.10.3", "jsonrpc-core", @@ -2146,6 +2211,7 @@ dependencies = [ "sodiumoxide", "sysinfo", "system-configuration", + "test_utilities", "thousands", "time 0.3.11", "tiny-bip39", @@ -2539,8 +2605,8 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b95af56fee93df76d721d356ac1ca41fccf168bc448eb14049234df764ba3e76" dependencies = [ - "proc-macro2 1.0.59", - "quote 1.0.28", + "proc-macro2 1.0.94", + "quote 1.0.40", "syn 1.0.85", ] @@ -2570,9 +2636,12 @@ checksum = "325a6d2ac5dee293c3b2612d4993b98aec1dff096b0a2dae70ed7d95784a05da" [[package]] name = "ppv-lite86" -version = "0.2.9" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c36fa947111f5c62a733b652544dd0016a43ce89619538a8ef92724a6f501a20" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] [[package]] name = "pretty-hex" @@ -2629,9 +2698,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.59" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6aeca18b86b413c660b781aa319e4e2648a3e6f9eadc9b47e9038e6fe9f3451b" +checksum = "a31971752e70b8b2686d7e46ec17fb38dad4051d94024c88df49b667caea9c84" dependencies = [ "unicode-ident", ] @@ -2653,11 +2722,11 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.28" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" dependencies = [ - "proc-macro2 1.0.59", + "proc-macro2 1.0.94", ] [[package]] @@ -3269,8 +3338,8 @@ version = "1.0.136" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9" dependencies = [ - "proc-macro2 1.0.59", - "quote 1.0.28", + "proc-macro2 1.0.94", + "quote 1.0.40", "syn 1.0.85", ] @@ -3314,8 +3383,8 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b2acd6defeddb41eb60bb468f8825d0cfd0c2a76bc03bfd235b6a1dc4f6a1ad5" dependencies = [ - "proc-macro2 1.0.59", - "quote 1.0.28", + "proc-macro2 1.0.94", + "quote 1.0.40", "syn 1.0.85", ] @@ -3532,19 +3601,30 @@ version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a684ac3dcd8913827e18cd09a68384ee66c1de24157e3c556c9ab16d85695fb7" dependencies = [ - "proc-macro2 1.0.59", - "quote 1.0.28", + "proc-macro2 1.0.94", + "quote 1.0.40", "unicode-xid 0.2.1", ] +[[package]] +name = "syn" +version = "2.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" +dependencies = [ + "proc-macro2 1.0.94", + "quote 1.0.40", + "unicode-ident", +] + [[package]] name = "synstructure" version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701" dependencies = [ - "proc-macro2 1.0.59", - "quote 1.0.28", + "proc-macro2 1.0.94", + "quote 1.0.40", "syn 1.0.85", "unicode-xid 0.2.1", ] @@ -3612,6 +3692,10 @@ dependencies = [ "phf_codegen", ] +[[package]] +name = "test_utilities" +version = "0.1.0" + [[package]] name = "textwrap" version = "0.11.0" @@ -3627,7 +3711,16 @@ version = "1.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.30", +] + +[[package]] +name = "thiserror" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +dependencies = [ + "thiserror-impl 2.0.12", ] [[package]] @@ -3636,11 +3729,22 @@ version = "1.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b" dependencies = [ - "proc-macro2 1.0.59", - "quote 1.0.28", + "proc-macro2 1.0.94", + "quote 1.0.40", "syn 1.0.85", ] +[[package]] +name = "thiserror-impl" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" +dependencies = [ + "proc-macro2 1.0.94", + "quote 1.0.40", + "syn 2.0.100", +] + [[package]] name = "thousands" version = "0.2.0" @@ -3689,7 +3793,7 @@ dependencies = [ "rand 0.7.3", "rustc-hash", "sha2 0.9.8", - "thiserror", + "thiserror 1.0.30", "unicode-normalization", "wasm-bindgen", "zeroize", @@ -4334,7 +4438,7 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aae2faf80ac463422992abf4de234731279c058aaf33171ca70277c98406b124" dependencies = [ - "quote 1.0.28", + "quote 1.0.40", "syn 1.0.85", ] @@ -4414,8 +4518,8 @@ dependencies = [ "bumpalo", "lazy_static", "log 0.4.18", - "proc-macro2 1.0.59", - "quote 1.0.28", + "proc-macro2 1.0.94", + "quote 1.0.40", "syn 1.0.85", "wasm-bindgen-shared", ] @@ -4438,7 +4542,7 @@ version = "0.2.78" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d56146e7c495528bf6587663bea13a8eb588d39b36b679d83972e1a2dbbdacf9" dependencies = [ - "quote 1.0.28", + "quote 1.0.40", "wasm-bindgen-macro-support", ] @@ -4448,8 +4552,8 @@ version = "0.2.78" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7803e0eea25835f8abdc585cd3021b3deb11543c6fe226dcd30b228857c5c5ab" dependencies = [ - "proc-macro2 1.0.59", - "quote 1.0.28", + "proc-macro2 1.0.94", + "quote 1.0.40", "syn 1.0.85", "wasm-bindgen-backend", "wasm-bindgen-shared", @@ -4714,6 +4818,26 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fc79f4a1e39857fc00c3f662cbf2651c771f00e9c15fe2abc341806bd46bd71" +[[package]] +name = "zerocopy" +version = "0.8.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be" +dependencies = [ + "proc-macro2 1.0.94", + "quote 1.0.40", + "syn 2.0.100", +] + [[package]] name = "zeroize" version = "1.4.3" @@ -4729,8 +4853,8 @@ version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65f1a51723ec88c66d5d1fe80c841f17f63587d6691901d66be9bec6c3b51f73" dependencies = [ - "proc-macro2 1.0.59", - "quote 1.0.28", + "proc-macro2 1.0.94", + "quote 1.0.40", "syn 1.0.85", "synstructure", ] diff --git a/node/Cargo.toml b/node/Cargo.toml index 7d01fd728..552761f32 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -29,6 +29,7 @@ futures = "0.1.31" heck = "0.3.3" http = "0.2.5" indoc = "1.0.3" +ip_country = { path = "../ip_country"} itertools = "0.10.1" lazy_static = "1.4.0" libc = "0.2.107" @@ -85,6 +86,7 @@ native-tls = {version = "0.2.8", features = ["vendored"]} simple-server = "0.4.0" serial_test_derive = "0.5.1" serial_test = "0.5.1" +test_utilities = { path = "../test_utilities"} trust-dns-proto = "0.8.0" [[bin]] diff --git a/node/src/apps.rs b/node/src/apps.rs index 66f2af1e6..2a38c7e47 100644 --- a/node/src/apps.rs +++ b/node/src/apps.rs @@ -6,7 +6,7 @@ use lazy_static::lazy_static; use masq_lib::constants::{HIGHEST_USABLE_PORT, LOWEST_USABLE_INSECURE_PORT}; use masq_lib::shared_schema::{ chain_arg, data_directory_arg, db_password_arg, real_user_arg, shared_app, ui_port_arg, - DB_PASSWORD_HELP, + DATA_DIRECTORY_HELP, DB_PASSWORD_HELP, }; use masq_lib::utils::DATA_DIRECTORY_DAEMON_HELP; @@ -36,7 +36,9 @@ pub fn app_daemon() -> App<'static, 'static> { } pub fn app_node() -> App<'static, 'static> { - shared_app(app_head().after_help(NODE_HELP_TEXT)).arg(ui_port_arg(&DAEMON_UI_PORT_HELP)) + shared_app(app_head().after_help(NODE_HELP_TEXT)) + .arg(data_directory_arg(DATA_DIRECTORY_HELP)) + .arg(ui_port_arg(&DAEMON_UI_PORT_HELP)) } pub fn app_config_dumper() -> App<'static, 'static> { diff --git a/node/src/bootstrapper.rs b/node/src/bootstrapper.rs index 2ef265e69..f49bb5707 100644 --- a/node/src/bootstrapper.rs +++ b/node/src/bootstrapper.rs @@ -15,6 +15,7 @@ use crate::json_discriminator_factory::JsonDiscriminatorFactory; use crate::listener_handler::ListenerHandler; use crate::listener_handler::ListenerHandlerFactory; use crate::listener_handler::ListenerHandlerFactoryReal; +use crate::neighborhood::node_location::get_node_location; use crate::neighborhood::DEFAULT_MIN_HOPS; use crate::node_configurator::node_configurator_standard::{ NodeConfiguratorStandardPrivileged, NodeConfiguratorStandardUnprivileged, @@ -52,7 +53,7 @@ use std::collections::HashMap; use std::env::var; use std::fmt; use std::fmt::{Debug, Display, Error, Formatter}; -use std::net::{Ipv4Addr, SocketAddr}; +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::path::PathBuf; use std::str::FromStr; use std::vec::Vec; @@ -505,6 +506,8 @@ impl ConfiguredByPrivilege for Bootstrapper { &alias_cryptde_null_opt, self.config.blockchain_bridge_config.chain, ); + // initialization of CountryFinder + let _ = get_node_location(Some(IpAddr::V4(Ipv4Addr::new(8, 8, 8, 8)))); let node_descriptor = Bootstrapper::make_local_descriptor( cryptdes.main, self.config.neighborhood_config.mode.node_addr_opt(), @@ -1207,7 +1210,7 @@ mod tests { main_cryptde_ref().public_key(), &NodeAddr::new(&IpAddr::from_str("1.2.3.4").unwrap(), &[5123]), Chain::EthRopsten, - main_cryptde_ref() + main_cryptde_ref(), )) ); TestLogHandler::new().exists_log_matching("INFO: Bootstrapper: MASQ Node local descriptor: masq://eth-ropsten:.+@1\\.2\\.3\\.4:5123"); @@ -1323,7 +1326,7 @@ mod tests { main_cryptde_ref().public_key(), &NodeAddr::new(&IpAddr::from_str("1.2.3.4").unwrap(), &[5123]), Chain::EthRopsten, - main_cryptde_ref() + main_cryptde_ref(), )) ); TestLogHandler::new().exists_log_matching("INFO: Bootstrapper: MASQ Node local descriptor: masq://eth-ropsten:.+@1\\.2\\.3\\.4:5123"); diff --git a/node/src/daemon/setup_reporter.rs b/node/src/daemon/setup_reporter.rs index 84c1db1e5..1fb47be60 100644 --- a/node/src/daemon/setup_reporter.rs +++ b/node/src/daemon/setup_reporter.rs @@ -26,7 +26,7 @@ use crate::sub_lib::neighborhood::NodeDescriptor; use crate::sub_lib::neighborhood::{NeighborhoodMode as NeighborhoodModeEnum, DEFAULT_RATE_PACK}; use crate::sub_lib::utils::make_new_multi_config; use crate::test_utils::main_cryptde; -use clap::value_t; +use clap::{value_t, App}; use itertools::Itertools; use masq_lib::blockchains::chains::Chain as BlockChain; use masq_lib::constants::DEFAULT_CHAIN; @@ -36,8 +36,10 @@ use masq_lib::messages::{UiSetupRequestValue, UiSetupResponseValue, UiSetupRespo use masq_lib::multi_config::{ CommandLineVcl, ConfigFileVcl, EnvironmentVcl, MultiConfig, VirtualCommandLine, }; -use masq_lib::shared_schema::{shared_app, ConfiguratorError}; -use masq_lib::utils::{add_chain_specific_directory, to_string, ExpectValue}; +use masq_lib::shared_schema::{data_directory_arg, shared_app, ConfiguratorError}; +use masq_lib::utils::{ + add_chain_specific_directory, to_string, ExpectValue, DATA_DIRECTORY_DAEMON_HELP, +}; use std::collections::HashMap; use std::fmt::Display; use std::net::{IpAddr, Ipv4Addr}; @@ -67,6 +69,10 @@ pub fn setup_cluster_from(input: Vec<(&str, &str, UiSetupResponseValueStatus)>) .collect::() } +fn setup_reporter_shared_app() -> App<'static, 'static> { + shared_app(app_head()).arg(data_directory_arg(DATA_DIRECTORY_DAEMON_HELP.as_str())) +} + pub trait SetupReporter { fn get_modified_setup( &self, @@ -222,7 +228,7 @@ impl SetupReporterReal { } pub fn get_default_params() -> SetupCluster { - let schema = shared_app(app_head()); + let schema = setup_reporter_shared_app(); schema .p .opts @@ -492,7 +498,7 @@ impl SetupReporterReal { environment: bool, config_file: bool, ) -> Result, ConfiguratorError> { - let app = shared_app(app_head()); + let app = setup_reporter_shared_app(); let mut vcls: Vec> = vec![]; if let Some(command_line) = command_line_opt.clone() { vcls.push(Box::new(CommandLineVcl::new(command_line))); diff --git a/node/src/lib.rs b/node/src/lib.rs index b13d219b1..cae5885d2 100644 --- a/node/src/lib.rs +++ b/node/src/lib.rs @@ -8,6 +8,8 @@ pub mod sub_lib; extern crate masq_lib; extern crate core; +extern crate ip_country_lib; + #[cfg(test)] mod node_test_utils; @@ -44,6 +46,6 @@ mod stream_messages; mod stream_reader; mod stream_writer_sorted; mod stream_writer_unsorted; -pub mod test_utils; //TODO we should make some effort for collections of testing utils to be really test conditioned. +pub mod test_utils; pub mod tls_discriminator_factory; pub mod ui_gateway; diff --git a/node/src/neighborhood/dot_graph.rs b/node/src/neighborhood/dot_graph.rs index 345afe077..78de9f339 100644 --- a/node/src/neighborhood/dot_graph.rs +++ b/node/src/neighborhood/dot_graph.rs @@ -12,6 +12,7 @@ pub struct NodeRenderableInner { pub version: u32, pub accepts_connections: bool, pub routes_data: bool, + pub country_code: String, } pub struct NodeRenderable { @@ -45,10 +46,11 @@ impl NodeRenderable { fn render_label(&self) -> String { let inner_string = match &self.inner { Some(inner) => format!( - "{}{} v{}\\n", + "{}{} v{} {}\\n", if inner.accepts_connections { "A" } else { "a" }, if inner.routes_data { "R" } else { "r" }, inner.version, + inner.country_code ), None => String::new(), }; @@ -107,6 +109,7 @@ mod tests { version: 1, accepts_connections: true, routes_data: true, + country_code: "ZZ".to_string(), }), public_key: public_key.clone(), node_addr: None, @@ -120,7 +123,7 @@ mod tests { assert_string_contains( &result, &format!( - "\"{}\" [label=\"AR v1\\n{}\"];", + "\"{}\" [label=\"AR v1 ZZ\\n{}\"];", public_key_64, public_key_trunc ), ); @@ -135,6 +138,7 @@ mod tests { version: 1, accepts_connections: false, routes_data: false, + country_code: "ZZ".to_string(), }), public_key: public_key.clone(), node_addr: None, @@ -148,7 +152,7 @@ mod tests { assert_string_contains( &result, &format!( - "\"{}\" [label=\"ar v1\\n{}\"];", + "\"{}\" [label=\"ar v1 ZZ\\n{}\"];", public_key_64, public_key_64 ), ); diff --git a/node/src/neighborhood/gossip.rs b/node/src/neighborhood/gossip.rs index cdda92fdc..465eb2749 100644 --- a/node/src/neighborhood/gossip.rs +++ b/node/src/neighborhood/gossip.rs @@ -5,7 +5,6 @@ use crate::neighborhood::dot_graph::{ render_dot_graph, DotRenderable, EdgeRenderable, NodeRenderable, NodeRenderableInner, }; use crate::neighborhood::neighborhood_database::NeighborhoodDatabase; -use crate::neighborhood::AccessibleGossipRecord; use crate::sub_lib::cryptde::{CryptDE, CryptData, PlainData, PublicKey}; use crate::sub_lib::hopper::MessageType; use crate::sub_lib::node_addr::NodeAddr; @@ -378,11 +377,16 @@ impl Gossip_0v1 { to: k.clone(), }) }); + let country_code = match &nri.country_code_opt { + Some(cc) => cc.clone(), + None => "ZZ".to_string(), + }; node_renderables.push(NodeRenderable { inner: Some(NodeRenderableInner { version: nri.version, accepts_connections: nri.accepts_connections, routes_data: nri.routes_data, + country_code, }), public_key: nri.public_key.clone(), node_addr: addr.clone(), @@ -460,6 +464,51 @@ impl<'a> GossipBuilder<'a> { } } +#[derive(Debug, PartialEq, Eq, Clone)] +pub struct AccessibleGossipRecord { + pub signed_gossip: PlainData, + pub signature: CryptData, + pub node_addr_opt: Option, + pub inner: NodeRecordInner_0v1, +} + +impl AccessibleGossipRecord { + pub fn regenerate_signed_gossip(&mut self, cryptde: &dyn CryptDE) { + let (signed_gossip, signature) = regenerate_signed_gossip(&self.inner, cryptde); + self.signed_gossip = signed_gossip; + self.signature = signature; + } +} + +impl TryFrom for AccessibleGossipRecord { + type Error = String; + + fn try_from(value: GossipNodeRecord) -> Result { + match serde_cbor::de::from_slice(value.signed_data.as_slice()) { + Ok(inner) => Ok(AccessibleGossipRecord { + signed_gossip: value.signed_data, + signature: value.signature, + node_addr_opt: value.node_addr_opt, + inner, + }), + Err(e) => Err(format!("{}", e)), + } + } +} + +pub fn regenerate_signed_gossip( + inner: &NodeRecordInner_0v1, + cryptde: &dyn CryptDE, // Must be the correct CryptDE for the Node from which inner came: used for signing +) -> (PlainData, CryptData) { + let signed_gossip = + PlainData::from(serde_cbor::ser::to_vec(&inner).expect("Serialization failed")); + let signature = match cryptde.sign(&signed_gossip) { + Ok(sig) => sig, + Err(e) => unimplemented!("TODO: Signing error: {:?}", e), + }; + (signed_gossip, signature) +} + #[cfg(test)] mod tests { use super::super::gossip::GossipBuilder; @@ -635,8 +684,8 @@ mod tests { "\n\tinner: NodeRecordInner_0v1 {\n\t\tpublic_key: 0x01020304,\n\t\tnode_addr_opt: Some(1.2.3.4:[1234]),\n\t\tearning_wallet: Wallet { kind: Address(0x546900db8d6e0937497133d1ae6fdf5f4b75bcd0) },\n\t\trate_pack: RatePack { routing_byte_rate: 1235, routing_service_rate: 1434, exit_byte_rate: 1237, exit_service_rate: 1634 },\n\t\tneighbors: [],\n\t\tversion: 2,\n\t},", "\n\tnode_addr_opt: Some(1.2.3.4:[1234]),", "\n\tsigned_data: -Length: 229 (0xe5) bytes -0000: a7 6a 70 75 62 6c 69 63 5f 6b 65 79 44 01 02 03 .jpublic_keyD... +Length: 249 (0xf9) bytes +0000: a8 6a 70 75 62 6c 69 63 5f 6b 65 79 44 01 02 03 .jpublic_keyD... 0010: 04 6e 65 61 72 6e 69 6e 67 5f 77 61 6c 6c 65 74 .nearning_wallet 0020: a1 67 61 64 64 72 65 73 73 94 18 54 18 69 00 18 .gaddress..T.i.. 0030: db 18 8d 18 6e 09 18 37 18 49 18 71 18 33 18 d1 ....n..7.I.q.3.. @@ -650,11 +699,12 @@ Length: 229 (0xe5) bytes 00b0: 6e 65 69 67 68 62 6f 72 73 80 73 61 63 63 65 70 neighbors.saccep 00c0: 74 73 5f 63 6f 6e 6e 65 63 74 69 6f 6e 73 f5 6b ts_connections.k 00d0: 72 6f 75 74 65 73 5f 64 61 74 61 f5 67 76 65 72 routes_data.gver -00e0: 73 69 6f 6e 02 sion.", +00e0: 73 69 6f 6e 02 70 63 6f 75 6e 74 72 79 5f 63 6f sion.pcountry_co +00f0: 64 65 5f 6f 70 74 62 41 44 de_optbAD", "\n\tsignature: Length: 24 (0x18) bytes -0000: 01 02 03 04 8a 7e b2 0e c4 ea fe d8 ac 3c 89 2d .....~.......<.- -0010: 2d f1 e9 76 d1 76 db 67 -..v.v.g" +0000: 01 02 03 04 9b be 7e b3 92 5d fe 69 9e c2 d3 86 ......~..].i.... +0010: 33 15 0d 91 3a 33 31 88 3...:31." ); assert_eq!(result, expected); @@ -691,7 +741,8 @@ Length: 4 (0x4) bytes source_node.inner.public_key = PublicKey::new(&b"ABCDEFGHIJKLMNOPQRSTUVWXYZ"[..]); let mut target_node = make_node_record(2345, true); target_node.inner.public_key = PublicKey::new(&b"ZYXWVUTSRQPONMLKJIHGFEDCBA"[..]); - let neighbor = make_node_record(3456, false); + let mut neighbor = make_node_record(3456, false); + neighbor.inner.country_code_opt = Some("FR".to_string()); let mut db = db_from_node(&source_node); db.add_node(target_node.clone()).unwrap(); db.add_node(neighbor.clone()).unwrap(); @@ -715,14 +766,14 @@ Length: 4 (0x4) bytes let result = gossip.to_dot_graph(&source_node, &target_node); assert_string_contains(&result, "digraph db { "); - assert_string_contains(&result, "\"AwQFBg\" [label=\"AR v0\\nAwQFBg\"]; "); + assert_string_contains(&result, "\"AwQFBg\" [label=\"AR v0 FR\\nAwQFBg\"]; "); assert_string_contains( &result, - "\"QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVo\" [label=\"AR v1\\nQUJDREVG\\n1.2.3.4:1234\"] [style=filled]; ", + "\"QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVo\" [label=\"AR v1 AD\\nQUJDREVG\\n1.2.3.4:1234\"] [style=filled]; ", ); assert_string_contains( &result, - "\"WllYV1ZVVFNSUVBPTk1MS0pJSEdGRURDQkE\" [label=\"AR v0\\nWllYV1ZV\\n2.3.4.5:2345\"] [shape=box]; ", + "\"WllYV1ZVVFNSUVBPTk1MS0pJSEdGRURDQkE\" [label=\"AR v0 AO\\nWllYV1ZV\\n2.3.4.5:2345\"] [shape=box]; ", ); assert_string_contains( &result, diff --git a/node/src/neighborhood/gossip_acceptor.rs b/node/src/neighborhood/gossip_acceptor.rs index 2548a7593..2badeb818 100644 --- a/node/src/neighborhood/gossip_acceptor.rs +++ b/node/src/neighborhood/gossip_acceptor.rs @@ -1,9 +1,11 @@ // Copyright (c) 2019, MASQ (https://masq.ai) and/or its affiliates. All rights reserved. -use crate::neighborhood::gossip::{GossipBuilder, Gossip_0v1}; +use crate::neighborhood::gossip::{ + AccessibleGossipRecord, GossipBuilder, GossipNodeRecord, Gossip_0v1, +}; use crate::neighborhood::neighborhood_database::{NeighborhoodDatabase, NeighborhoodDatabaseError}; use crate::neighborhood::node_record::NodeRecord; -use crate::neighborhood::AccessibleGossipRecord; +use crate::neighborhood::UserExitPreferences; use crate::sub_lib::cryptde::{CryptDE, PublicKey}; use crate::sub_lib::neighborhood::{ ConnectionProgressEvent, ConnectionProgressMessage, GossipFailure_0v1, NeighborhoodMetadata, @@ -91,6 +93,23 @@ impl GossipHandler for DebutHandler { if database.node_by_key(&agrs[0].inner.public_key).is_some() { return Qualification::Unmatched; } + // TODO create optimization card: drive in the test and following commented out code, + // TODO: Imagine a brand-new network, consisting only of Node A. + // When Node B debuts, Node A cannot respond with an Introduction, + // since there's nobody to introduce. Therefore, Node A must + // respond with a single-Node Gossip that will currently be + // interpreted as a Debut by Node B, resulting in another + // single-Node Gossip from B to A. This last Gossip isn't a + // problem, because Node A will ignore it since B is already + // in its database. However, there is a possible optimization: + // drive in the code below, and Node B will no longer interpret + // Node A's acceptance Gossip as another Debut, because it will + // see that Node A already has Node B as a neighbor. This means + // Node A's original response will be interpreted as Standard + // Gossip. + // if agrs[0].inner.neighbors.contains(database.root_key()) { + // return Qualification::Unmatched; + // } match &agrs[0].node_addr_opt { None => { if agrs[0].inner.accepts_connections { @@ -130,7 +149,7 @@ impl GossipHandler for DebutHandler { database: &mut NeighborhoodDatabase, mut agrs: Vec, gossip_source: SocketAddr, - _neighborhood_metadata: NeighborhoodMetadata, + neighborhood_metadata: NeighborhoodMetadata, ) -> GossipAcceptanceResult { let source_agr = { let mut agr = agrs.remove(0); // empty Gossip shouldn't get here @@ -165,7 +184,13 @@ impl GossipHandler for DebutHandler { source_node_addr, ); } - if let Ok(result) = self.try_accept_debut(cryptde, database, &source_agr, gossip_source) { + if let Ok(result) = self.try_accept_debut( + cryptde, + database, + &source_agr, + gossip_source, + neighborhood_metadata.user_exit_preferences_opt, + ) { return result; } debug!(self.logger, "Seeking neighbor for Pass"); @@ -240,7 +265,7 @@ impl DebutHandler { None => { debug!( self.logger, - "No degree-3-or-greater neighbors; can't find more-appropriate neighbor" + "No degree-3-or-greater neighbors; can't find more-appropriate neighbor.", ); None } @@ -266,13 +291,20 @@ impl DebutHandler { database: &mut NeighborhoodDatabase, debuting_agr: &AccessibleGossipRecord, gossip_source: SocketAddr, + user_exit_preferences_opt: Option, ) -> Result { if database.gossip_target_degree(database.root().public_key()) >= MAX_DEGREE { debug!(self.logger, "Neighbor count already at maximum"); return Err(()); } let debut_node_addr_opt = debuting_agr.node_addr_opt.clone(); - let debuting_node = NodeRecord::from(debuting_agr); + let mut debuting_node = NodeRecord::from(debuting_agr); + match user_exit_preferences_opt { + Some(user_exit_preferences) => { + user_exit_preferences.assign_nodes_country_undesirability(&mut debuting_node); + } + None => (), + } let debut_node_key = database .add_node(debuting_node) .expect("Debuting Node suddenly appeared in database"); @@ -289,12 +321,12 @@ impl DebutHandler { root_mut.increment_version(); root_mut.regenerate_signed_gossip(cryptde); trace!(self.logger, "Current database: {}", database.to_dot_graph()); - if Self::should_not_make_introduction(debuting_agr) { + if Self::should_not_make_another_introduction(debuting_agr) { let ip_addr_str = match &debuting_agr.node_addr_opt { Some(node_addr) => node_addr.ip_addr().to_string(), None => "?.?.?.?".to_string(), }; - debug!(self.logger, "Node {} at {} is responding to first introduction: sending update Gossip instead of further introduction", + debug!(self.logger, "Node {} at {} is responding to first introduction: sending standard Gossip instead of further introduction", debuting_agr.inner.public_key, ip_addr_str); Ok(GossipAcceptanceResult::Accepted) @@ -310,11 +342,29 @@ impl DebutHandler { None => { debug!( self.logger, - "DebutHandler can't make an introduction, but is accepting {} at {} and broadcasting change", + "DebutHandler has no one to introduce, but is debuting back to {} at {}", &debut_node_key, gossip_source, ); - Ok(GossipAcceptanceResult::Accepted) + trace!( + self.logger, + "DebutHandler database state: {}", + &database.to_dot_graph(), + ); + let debut_gossip = Self::create_debut_gossip_response( + cryptde, + database, + debut_node_key, + ); + Ok(GossipAcceptanceResult::Reply( + debut_gossip, + debuting_agr.inner.public_key.clone(), + debuting_agr + .node_addr_opt + .as_ref() + .expect("Debut gossip always has an IP") + .clone(), + )) } } } @@ -323,6 +373,28 @@ impl DebutHandler { } } + fn create_debut_gossip_response( + cryptde: &dyn CryptDE, + database: &NeighborhoodDatabase, + debut_node_key: PublicKey, + ) -> Gossip_0v1 { + let mut root_node = database.root().clone(); + root_node.clear_half_neighbors(); + root_node + .add_half_neighbor_key(debut_node_key.clone()) + .unwrap_or_else(|e| { + panic!( + "Couldn't add debuting {} as a half neighbor: {:?}", + debut_node_key, e + ) + }); + let root_node_addr_opt = root_node.node_addr_opt(); + let gnr = GossipNodeRecord::from((root_node.inner, root_node_addr_opt, cryptde)); + Gossip_0v1 { + node_records: vec![gnr], + } + } + fn make_introduction( &self, database: &NeighborhoodDatabase, @@ -443,7 +515,7 @@ impl DebutHandler { keys } - fn should_not_make_introduction(debuting_agr: &AccessibleGossipRecord) -> bool { + fn should_not_make_another_introduction(debuting_agr: &AccessibleGossipRecord) -> bool { !debuting_agr.inner.neighbors.is_empty() } @@ -645,7 +717,12 @@ impl GossipHandler for IntroductionHandler { .as_ref() .expect("IP Address not found for the Node Addr.") .ip_addr(); - match self.update_database(database, cryptde, introducer) { + match self.update_database( + database, + cryptde, + introducer, + neighborhood_metadata.user_exit_preferences_opt, + ) { Ok(_) => (), Err(e) => { return GossipAcceptanceResult::Ban(format!( @@ -805,6 +882,7 @@ impl IntroductionHandler { database: &mut NeighborhoodDatabase, cryptde: &dyn CryptDE, introducer: AccessibleGossipRecord, + user_exit_preferences_opt: Option, ) -> Result { let introducer_key = introducer.inner.public_key.clone(); match database.node_by_key_mut(&introducer_key) { @@ -829,7 +907,12 @@ impl IntroductionHandler { } } None => { - let new_introducer = NodeRecord::from(introducer); + let mut new_introducer = NodeRecord::from(introducer); + match user_exit_preferences_opt { + Some(user_exit_preferences) => user_exit_preferences + .assign_nodes_country_undesirability(&mut new_introducer), + None => (), + } debug!( self.logger, "Adding introducer {} to database", introducer_key @@ -944,6 +1027,7 @@ impl GossipHandler for StandardGossipHandler { database, &filtered_agrs, gossip_source, + neighborhood_metadata.user_exit_preferences_opt.as_ref(), ); db_changed = self.identify_and_update_obsolete_nodes(database, filtered_agrs) || db_changed; db_changed = @@ -1055,6 +1139,7 @@ impl StandardGossipHandler { database: &mut NeighborhoodDatabase, agrs: &[AccessibleGossipRecord], gossip_source: SocketAddr, + user_exit_preferences_opt: Option<&UserExitPreferences>, ) -> bool { let all_keys = database .keys() @@ -1072,7 +1157,13 @@ impl StandardGossipHandler { } }) .for_each(|agr| { - let node_record = NodeRecord::from(agr); + let mut node_record = NodeRecord::from(agr); + match user_exit_preferences_opt { + Some(user_exit_preferences) => { + user_exit_preferences.assign_nodes_country_undesirability(&mut node_record) + } + None => (), + } trace!( self.logger, "Discovered new Node {:?}: {:?}", @@ -1329,6 +1420,9 @@ mod tests { use crate::neighborhood::gossip_producer::GossipProducer; use crate::neighborhood::gossip_producer::GossipProducerReal; use crate::neighborhood::node_record::NodeRecord; + use crate::neighborhood::{ + FallbackPreference, UserExitPreferences, UNREACHABLE_COUNTRY_PENALTY, + }; use crate::sub_lib::cryptde_null::CryptDENull; use crate::sub_lib::neighborhood::{ConnectionProgressEvent, ConnectionProgressMessage}; use crate::sub_lib::utils::time_t_timestamp; @@ -1340,9 +1434,12 @@ mod tests { use crate::test_utils::unshared_test_utils::make_cpm_recipient; use crate::test_utils::{assert_contains, main_cryptde, vec_to_set}; use actix::System; + use itertools::Itertools; + use masq_lib::messages::ExitLocation; use masq_lib::test_utils::logging::{init_test_logging, TestLogHandler}; use masq_lib::test_utils::utils::TEST_DEFAULT_CHAIN; use std::convert::TryInto; + use std::net::Ipv4Addr; use std::ops::{Add, Sub}; use std::str::FromStr; use std::time::Duration; @@ -1366,6 +1463,7 @@ mod tests { connection_progress_peers: vec![], cpm_recipient: make_cpm_recipient().0, db_patch_size: DB_PATCH_SIZE_FOR_TEST, + user_exit_preferences_opt: None, } } @@ -1625,6 +1723,52 @@ mod tests { assert_eq!(result, GossipAcceptanceResult::Accepted); } + #[test] + fn two_parallel_debuts_in_progress_handled_by_try_accept_debut_without_introduction() { + let root_node = make_node_record(1234, true); + let half_neighbor_debutant = make_node_record(2345, true); + let new_debutant = make_node_record(4567, true); + let root_node_cryptde = CryptDENull::from(&root_node.public_key(), TEST_DEFAULT_CHAIN); + let mut dest_db = db_from_node(&root_node); + dest_db.add_node(half_neighbor_debutant.clone()).unwrap(); + dest_db.add_arbitrary_half_neighbor( + root_node.public_key(), + half_neighbor_debutant.public_key(), + ); + let logger = Logger::new("Debut test"); + let subject = DebutHandler::new(logger); + let neighborhood_metadata = make_default_neighborhood_metadata(); + + let counter_debut = subject + .try_accept_debut( + &root_node_cryptde, + &mut dest_db, + &AccessibleGossipRecord::from(&new_debutant), + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(4, 5, 6, 7)), 4567), + neighborhood_metadata.user_exit_preferences_opt, + ) + .unwrap(); + + let (debut_reply, dest_public_key, dest_node_addr) = match counter_debut { + GossipAcceptanceResult::Reply( + ref debut_reply, + ref dest_public_key, + ref dest_node_addr, + ) => (debut_reply, dest_public_key, dest_node_addr), + x => panic!("Expected Reply, got {:?}", x), + }; + assert_eq!(dest_public_key, new_debutant.public_key()); + assert_eq!(dest_node_addr, &new_debutant.node_addr_opt().unwrap()); + assert_eq!( + counter_debut, + GossipAcceptanceResult::Reply( + debut_reply.clone(), + dest_public_key.clone(), + dest_node_addr.clone() + ) + ) + } + #[test] fn proper_pass_is_identified_and_processed() { let (gossip, pass_target, gossip_source) = make_pass(2345); @@ -1934,6 +2078,16 @@ mod tests { let cryptde = CryptDENull::from(dest_db.root().public_key(), TEST_DEFAULT_CHAIN); let subject = IntroductionHandler::new(Logger::new("test")); let agrs: Vec = gossip.try_into().unwrap(); + let mut neighborhood_metadata = make_default_neighborhood_metadata(); + neighborhood_metadata.user_exit_preferences_opt = Some(UserExitPreferences { + exit_countries: vec!["FR".to_string()], + fallback_preference: FallbackPreference::ExitCountryNoFallback, + locations_opt: Some(vec![ExitLocation { + country_codes: vec!["FR".to_string()], + priority: 2, + }]), + db_countries: vec!["FR".to_string()], + }); let qualifies_result = subject.qualifies(&dest_db, &agrs, gossip_source); let handle_result = subject.handle( @@ -1941,7 +2095,7 @@ mod tests { &mut dest_db, agrs.clone(), gossip_source, - make_default_neighborhood_metadata(), + neighborhood_metadata, ); assert_eq!(Qualification::Matched, qualifies_result); @@ -1956,19 +2110,22 @@ mod tests { ), handle_result ); - let expected_introducer = NodeRecord::from(&agrs[0]); - assert_eq!( - Some(&expected_introducer), - dest_db.node_by_key(&agrs[0].inner.public_key) - ); + + let result_introducer: &NodeRecord = + dest_db.node_by_key(&agrs[0].inner.public_key).unwrap(); + let mut expected_introducer = NodeRecord::from(&agrs[0]); + expected_introducer.metadata.last_update = result_introducer.metadata.last_update; + expected_introducer.metadata.country_undesirability = 0; + expected_introducer.resign(); + assert_eq!(result_introducer, &expected_introducer); assert_eq!( - true, dest_db .root() - .has_half_neighbor(expected_introducer.public_key()) + .has_half_neighbor(expected_introducer.public_key()), + true ); - assert_eq!(1, dest_db.root().version()); - assert_eq!(None, dest_db.node_by_key(&agrs[1].inner.public_key)); + assert_eq!(dest_db.root().version(), 1); + assert_eq!(dest_db.node_by_key(&agrs[1].inner.public_key), None); } #[test] @@ -2032,11 +2189,12 @@ mod tests { ), handle_result ); - let expected_introducer = NodeRecord::from(&agrs[0]); - assert_eq!( - Some(&expected_introducer), - dest_db.node_by_key(&agrs[0].inner.public_key) - ); + let result_introducer: &NodeRecord = + dest_db.node_by_key(&agrs[0].inner.public_key).unwrap(); + let mut expected_introducer = NodeRecord::from(&agrs[0]); + expected_introducer.metadata.last_update = result_introducer.metadata.last_update; + expected_introducer.resign(); + assert_eq!(result_introducer, &expected_introducer); assert_eq!( true, dest_db @@ -2087,11 +2245,13 @@ mod tests { ), handle_result ); - let expected_introducer = NodeRecord::from(&agrs[0]); - assert_eq!( - Some(&expected_introducer), - dest_db.node_by_key(&agrs[0].inner.public_key) - ); + + let result_introducer: &NodeRecord = + dest_db.node_by_key(&agrs[0].inner.public_key).unwrap(); + let mut expected_introducer = NodeRecord::from(&agrs[0]); + expected_introducer.metadata.last_update = result_introducer.metadata.last_update; + expected_introducer.resign(); + assert_eq!(result_introducer, &expected_introducer); assert_eq!( true, dest_db @@ -2289,17 +2449,17 @@ mod tests { let src_root = make_node_record(1234, true); let dest_root = make_node_record(2345, true); let mut src_db = db_from_node(&src_root); - let node_a = make_node_record(3456, true); - let node_b = make_node_record(4567, true); + let node_a_ao = make_node_record(5678, true); + let node_b_ad = make_node_record(1235, true); let mut dest_db = db_from_node(&dest_root); dest_db.add_node(src_root.clone()).unwrap(); dest_db.add_arbitrary_full_neighbor(dest_root.public_key(), src_root.public_key()); src_db.add_node(dest_db.root().clone()).unwrap(); - src_db.add_node(node_a.clone()).unwrap(); - src_db.add_node(node_b.clone()).unwrap(); + src_db.add_node(node_a_ao.clone()).unwrap(); + src_db.add_node(node_b_ad.clone()).unwrap(); src_db.add_arbitrary_full_neighbor(src_root.public_key(), dest_root.public_key()); - src_db.add_arbitrary_half_neighbor(src_root.public_key(), &node_a.public_key()); - src_db.add_arbitrary_full_neighbor(src_root.public_key(), &node_b.public_key()); + src_db.add_arbitrary_half_neighbor(src_root.public_key(), &node_a_ao.public_key()); + src_db.add_arbitrary_full_neighbor(src_root.public_key(), &node_b_ad.public_key()); src_db .node_by_key_mut(src_root.public_key()) .unwrap() @@ -2307,8 +2467,8 @@ mod tests { src_db.resign_node(src_root.public_key()); let gossip = GossipBuilder::new(&src_db) .node(src_root.public_key(), true) - .node(node_a.public_key(), false) - .node(node_b.public_key(), false) + .node(node_a_ao.public_key(), false) + .node(node_b_ad.public_key(), false) .build(); let subject = StandardGossipHandler::new(Logger::new("test")); let cryptde = CryptDENull::from(dest_db.root().public_key(), TEST_DEFAULT_CHAIN); @@ -2316,6 +2476,15 @@ mod tests { let gossip_source: SocketAddr = src_root.node_addr_opt().unwrap().into(); let (cpm_recipient, recording_arc) = make_cpm_recipient(); let mut neighborhood_metadata = make_default_neighborhood_metadata(); + neighborhood_metadata.user_exit_preferences_opt = Some(UserExitPreferences { + exit_countries: vec!["AO".to_string()], + fallback_preference: FallbackPreference::ExitCountryWithFallback, + locations_opt: Some(vec![ExitLocation { + country_codes: vec!["AO".to_string()], + priority: 1, + }]), + db_countries: vec!["AO".to_string()], + }); neighborhood_metadata.cpm_recipient = cpm_recipient; let system = System::new("test"); @@ -2328,6 +2497,22 @@ mod tests { neighborhood_metadata, ); + assert_eq!( + dest_db + .node_by_key(node_a_ao.public_key()) + .unwrap() + .metadata + .country_undesirability, + 0u32 + ); + assert_eq!( + dest_db + .node_by_key(node_b_ad.public_key()) + .unwrap() + .metadata + .country_undesirability, + UNREACHABLE_COUNTRY_PENALTY + ); assert_eq!(Qualification::Matched, qualifies_result); assert_eq!(GossipAcceptanceResult::Accepted, handle_result); assert_eq!( @@ -2336,12 +2521,12 @@ mod tests { ); assert!(dest_db.has_full_neighbor(dest_db.root().public_key(), src_db.root().public_key())); assert_eq!( - &src_db.node_by_key(node_a.public_key()).unwrap().inner, - &dest_db.node_by_key(node_a.public_key()).unwrap().inner + &src_db.node_by_key(node_a_ao.public_key()).unwrap().inner, + &dest_db.node_by_key(node_a_ao.public_key()).unwrap().inner ); assert_eq!( - &src_db.node_by_key(node_b.public_key()).unwrap().inner, - &dest_db.node_by_key(node_b.public_key()).unwrap().inner + &src_db.node_by_key(node_b_ad.public_key()).unwrap().inner, + &dest_db.node_by_key(node_b_ad.public_key()).unwrap().inner ); System::current().stop(); assert_eq!(system.run(), 0); @@ -2884,32 +3069,59 @@ mod tests { fn first_debut_is_handled() { let mut root_node = make_node_record(1234, true); let root_node_cryptde = CryptDENull::from(&root_node.public_key(), TEST_DEFAULT_CHAIN); - let mut dest_db = db_from_node(&root_node); - let (gossip, debut_node, gossip_source) = make_debut(2345, Mode::Standard); + let mut source_db = db_from_node(&root_node); + let (gossip, mut debut_node, gossip_source) = make_debut(2345, Mode::Standard); //debut node is FR + let mut expected_source_db = db_from_node(&root_node); + expected_source_db + .add_arbitrary_half_neighbor(root_node.public_key(), debut_node.public_key()); + expected_source_db.root_mut().inner.version = 1; + expected_source_db.root_mut().resign(); + let expected_gossip_response = GossipBuilder::new(&expected_source_db) + .node(root_node.public_key(), true) + .build(); let subject = make_subject(&root_node_cryptde); + let mut neighborhood_metadata = make_default_neighborhood_metadata(); + neighborhood_metadata.user_exit_preferences_opt = Some(UserExitPreferences { + exit_countries: vec!["CZ".to_string()], + fallback_preference: FallbackPreference::ExitCountryWithFallback, + locations_opt: Some(vec![ExitLocation { + country_codes: vec!["CZ".to_string()], + priority: 1, + }]), + db_countries: vec!["CZ".to_string()], + }); let before = time_t_timestamp(); let result = subject.handle( - &mut dest_db, + &mut source_db, gossip.try_into().unwrap(), gossip_source, - make_default_neighborhood_metadata(), + neighborhood_metadata, ); let after = time_t_timestamp(); - assert_eq!(GossipAcceptanceResult::Accepted, result); + let expected_result = GossipAcceptanceResult::Reply( + expected_gossip_response, + debut_node.public_key().clone(), + debut_node.node_addr_opt().unwrap(), + ); + assert_eq!(result, expected_result); root_node .add_half_neighbor_key(debut_node.public_key().clone()) .unwrap(); root_node.increment_version(); + root_node.metadata.last_update = source_db.root().metadata.last_update; root_node.resign(); - assert_eq!(&root_node, dest_db.root()); - assert_node_records_eq( - dest_db.node_by_key_mut(debut_node.public_key()).unwrap(), - &debut_node, - before, - after, + assert_eq!(&root_node, source_db.root()); + let reference_node = source_db.node_by_key_mut(debut_node.public_key()).unwrap(); + debut_node.metadata.last_update = reference_node.metadata.last_update; + debut_node.resign(); + assert_eq!( + reference_node.metadata.country_undesirability, + UNREACHABLE_COUNTRY_PENALTY ); + reference_node.metadata.country_undesirability = 0u32; + assert_node_records_eq(reference_node, &debut_node, before, after); } #[test] @@ -2924,7 +3136,7 @@ mod tests { .unwrap() .resign(); dest_db.node_by_key_mut(existing_node_key).unwrap().resign(); - let (gossip, debut_node, gossip_source) = make_debut(2345, Mode::Standard); + let (gossip, mut debut_node, gossip_source) = make_debut(2345, Mode::Standard); let subject = make_subject(&root_node_cryptde); let before = time_t_timestamp(); @@ -2955,14 +3167,13 @@ mod tests { .add_half_neighbor_key(existing_node_key.clone()) .unwrap(); root_node.increment_version(); + root_node.metadata.last_update = dest_db.root().metadata.last_update; root_node.resign(); assert_eq!(&root_node, dest_db.root()); - assert_node_records_eq( - dest_db.node_by_key_mut(debut_node.public_key()).unwrap(), - &debut_node, - before, - after, - ) + let reference_node = dest_db.node_by_key(debut_node.public_key()).unwrap(); + debut_node.metadata.last_update = reference_node.metadata.last_update; + debut_node.resign(); + assert_node_records_eq(reference_node, &debut_node, before, after) } #[test] @@ -2995,7 +3206,7 @@ mod tests { .unwrap() .resign(); - let (gossip, debut_node, gossip_source) = make_debut(2345, Mode::Standard); + let (gossip, mut debut_node, gossip_source) = make_debut(2345, Mode::Standard); let subject = make_subject(&root_node_cryptde); let before = time_t_timestamp(); @@ -3045,14 +3256,13 @@ mod tests { .add_half_neighbor_key(existing_node_3_key.clone()) .unwrap(); root_node.increment_version(); + root_node.metadata.last_update = dest_db.root().metadata.last_update; root_node.resign(); assert_eq!(&root_node, dest_db.root()); - assert_node_records_eq( - dest_db.node_by_key_mut(debut_node.public_key()).unwrap(), - &debut_node, - before, - after, - ) + let reference_node = dest_db.node_by_key(debut_node.public_key()).unwrap(); + debut_node.metadata.last_update = reference_node.metadata.last_update; + debut_node.resign(); + assert_node_records_eq(reference_node, &debut_node, before, after) } #[test] @@ -3135,6 +3345,7 @@ mod tests { root_node .add_half_neighbor_key(existing_node_4_key.clone()) .unwrap(); + root_node.metadata.last_update = dest_db.root().metadata.last_update; root_node.resign(); assert_eq!(&root_node, dest_db.root()); } @@ -3218,6 +3429,7 @@ mod tests { .add_half_neighbor_key(existing_node_5_key.clone()) .unwrap(); root_node.resign(); + root_node.metadata.last_update = dest_db.root().metadata.last_update; assert_eq!(&root_node, dest_db.root()); } @@ -3318,7 +3530,21 @@ mod tests { make_default_neighborhood_metadata(), ); - assert_eq!(result, GossipAcceptanceResult::Accepted); + let mut root_node = dest_db.root().clone(); + root_node.clear_half_neighbors(); + root_node + .add_half_neighbor_key(src_node.public_key().clone()) + .expect("expected half neighbor"); + let gnr = GossipNodeRecord::from(( + root_node.inner.clone(), + root_node.node_addr_opt(), + main_cryptde(), + )); + let debut_gossip = Gossip_0v1 { + node_records: vec![gnr], + }; + let expected = make_expected_non_introduction_debut_response(&src_node, debut_gossip); + assert_eq!(result, expected); assert_eq!( dest_db .node_by_key(dest_node.public_key()) @@ -3353,7 +3579,21 @@ mod tests { make_default_neighborhood_metadata(), ); - assert_eq!(result, GossipAcceptanceResult::Accepted); + let mut root_node = dest_db.root().clone(); + root_node.clear_half_neighbors(); + root_node + .add_half_neighbor_key(src_node.public_key().clone()) + .expect("expected half neighbor"); + let gnr = GossipNodeRecord::from(( + root_node.inner.clone(), + root_node.node_addr_opt(), + main_cryptde(), + )); + let debut_gossip = Gossip_0v1 { + node_records: vec![gnr], + }; + let expected = make_expected_non_introduction_debut_response(&src_node, debut_gossip); + assert_eq!(result, expected); assert_eq!( dest_db .node_by_key(dest_node.public_key()) @@ -3363,6 +3603,17 @@ mod tests { ); } + fn make_expected_non_introduction_debut_response( + src_node: &NodeRecord, + debut_gossip: Gossip_0v1, + ) -> GossipAcceptanceResult { + GossipAcceptanceResult::Reply( + debut_gossip, + src_node.public_key().clone(), + src_node.node_addr_opt().as_ref().unwrap().clone(), + ) + } + #[test] fn introduction_gossip_handler_sends_cpm_for_neighborship_established() { let cryptde = main_cryptde(); @@ -3652,6 +3903,7 @@ mod tests { let node_d = make_node_record(5678, false); let node_e = make_node_record(6789, true); let node_f = make_node_record(7890, true); + dest_db.add_node(node_a.clone()).unwrap(); dest_db.add_node(node_b.clone()).unwrap(); dest_db.add_node(node_d.clone()).unwrap(); @@ -3711,8 +3963,14 @@ mod tests { &mut expected_dest_db, vec![&node_a, &node_b, &node_d, &node_e, &node_f], ); + fix_nodes_last_updates(&mut expected_dest_db, &dest_db); + expected_dest_db + .node_by_key_mut(node_c.public_key()) + .unwrap() + .metadata + .node_location_opt = None; assert_node_records_eq( - dest_db.node_by_key_mut(root_node.public_key()).unwrap(), + dest_db.node_by_key(root_node.public_key()).unwrap(), expected_dest_db .node_by_key(root_node.public_key()) .unwrap(), @@ -3720,25 +3978,25 @@ mod tests { after, ); assert_node_records_eq( - dest_db.node_by_key_mut(node_a.public_key()).unwrap(), + dest_db.node_by_key(node_a.public_key()).unwrap(), expected_dest_db.node_by_key(node_a.public_key()).unwrap(), before, after, ); assert_node_records_eq( - dest_db.node_by_key_mut(node_b.public_key()).unwrap(), + dest_db.node_by_key(node_b.public_key()).unwrap(), expected_dest_db.node_by_key(node_b.public_key()).unwrap(), before, after, ); assert_node_records_eq( - dest_db.node_by_key_mut(node_c.public_key()).unwrap(), + dest_db.node_by_key(node_c.public_key()).unwrap(), expected_dest_db.node_by_key(node_c.public_key()).unwrap(), before, after, ); assert_node_records_eq( - dest_db.node_by_key_mut(node_d.public_key()).unwrap(), + dest_db.node_by_key(node_d.public_key()).unwrap(), expected_dest_db.node_by_key(node_d.public_key()).unwrap(), before, after, @@ -3747,6 +4005,29 @@ mod tests { assert_eq!(dest_db.node_by_key(node_f.public_key()), None); } + fn fix_nodes_last_updates( + expected_db: &mut NeighborhoodDatabase, + dest_db: &NeighborhoodDatabase, + ) { + let keys = expected_db + .keys() + .iter() + .map(|key| (*key).clone()) + .collect_vec(); + keys.into_iter() + .for_each(|pubkey| match dest_db.node_by_key(&pubkey) { + Some(node_record) => { + expected_db + .node_by_key_mut(&pubkey) + .unwrap() + .metadata + .last_update = node_record.metadata.last_update; + expected_db.node_by_key_mut(&pubkey).unwrap().resign(); + } + None => {} + }); + } + #[test] fn initial_standard_gossip_does_not_produce_neighborship_if_destination_degree_is_already_full() { @@ -3833,8 +4114,9 @@ mod tests { dest_node_mut.increment_version(); dest_node_mut.resign(); assert_eq!(result, GossipAcceptanceResult::Accepted); + fix_nodes_last_updates(&mut expected_dest_db, &dest_db); assert_node_records_eq( - dest_db.node_by_key_mut(third_node.public_key()).unwrap(), + dest_db.node_by_key(third_node.public_key()).unwrap(), expected_dest_db .node_by_key(third_node.public_key()) .unwrap(), @@ -3842,13 +4124,13 @@ mod tests { after, ); assert_node_records_eq( - dest_db.node_by_key_mut(src_node.public_key()).unwrap(), + dest_db.node_by_key(src_node.public_key()).unwrap(), expected_dest_db.node_by_key(src_node.public_key()).unwrap(), before, after, ); assert_node_records_eq( - dest_db.node_by_key_mut(dest_node.public_key()).unwrap(), + dest_db.node_by_key(dest_node.public_key()).unwrap(), expected_dest_db .node_by_key(dest_node.public_key()) .unwrap(), @@ -3905,7 +4187,7 @@ mod tests { let after = time_t_timestamp(); assert_eq!(result, GossipAcceptanceResult::Ignored); assert_node_records_eq( - dest_db.node_by_key_mut(dest_root.public_key()).unwrap(), + dest_db.node_by_key(dest_root.public_key()).unwrap(), original_dest_db .node_by_key(dest_root.public_key()) .unwrap(), @@ -3913,13 +4195,13 @@ mod tests { after, ); assert_node_records_eq( - dest_db.node_by_key_mut(src_root.public_key()).unwrap(), + dest_db.node_by_key(src_root.public_key()).unwrap(), original_dest_db.node_by_key(src_root.public_key()).unwrap(), before, after, ); assert_node_records_eq( - dest_db.node_by_key_mut(current_node.public_key()).unwrap(), + dest_db.node_by_key(current_node.public_key()).unwrap(), original_dest_db .node_by_key(current_node.public_key()) .unwrap(), @@ -3927,7 +4209,7 @@ mod tests { after, ); assert_node_records_eq( - dest_db.node_by_key_mut(obsolete_node.public_key()).unwrap(), + dest_db.node_by_key(obsolete_node.public_key()).unwrap(), original_dest_db .node_by_key(obsolete_node.public_key()) .unwrap(), @@ -4260,12 +4542,7 @@ mod tests { GossipAcceptorReal::new(crypt_de) } - fn assert_node_records_eq( - actual: &mut NodeRecord, - expected: &NodeRecord, - before: u32, - after: u32, - ) { + fn assert_node_records_eq(actual: &NodeRecord, expected: &NodeRecord, before: u32, after: u32) { assert!( actual.metadata.last_update >= before, "Timestamp should have been at least {}, but was {}", @@ -4278,7 +4555,6 @@ mod tests { after, actual.metadata.last_update ); - actual.metadata.last_update = expected.metadata.last_update; assert_eq!(actual, expected); } } diff --git a/node/src/neighborhood/gossip_producer.rs b/node/src/neighborhood/gossip_producer.rs index 5f5ad9c92..1ebd2777d 100644 --- a/node/src/neighborhood/gossip_producer.rs +++ b/node/src/neighborhood/gossip_producer.rs @@ -102,9 +102,9 @@ impl GossipProducerReal { mod tests { use super::super::gossip::GossipNodeRecord; use super::*; + use crate::neighborhood::gossip::AccessibleGossipRecord; use crate::neighborhood::neighborhood_database::ISOLATED_NODE_GRACE_PERIOD_SECS; use crate::neighborhood::node_record::{NodeRecord, NodeRecordInner_0v1}; - use crate::neighborhood::AccessibleGossipRecord; use crate::sub_lib::cryptde::CryptDE; use crate::sub_lib::cryptde_null::CryptDENull; use crate::sub_lib::utils::time_t_timestamp; @@ -336,7 +336,8 @@ mod tests { #[test] fn produce_debut_creates_a_gossip_to_a_target_about_ourselves_when_accepting_connections() { - let our_node_record: NodeRecord = make_node_record(7771, true); + let mut our_node_record: NodeRecord = make_node_record(7771, true); + our_node_record.inner.country_code_opt = None; let db = db_from_node(&our_node_record); let subject = GossipProducerReal::new(); @@ -374,6 +375,7 @@ mod tests { let result_gossip_record = result_gossip.node_records.first().unwrap(); assert_eq!(result_gossip_record.node_addr_opt, None); let result_node_record_inner = NodeRecordInner_0v1::try_from(result_gossip_record).unwrap(); + our_node_record.inner.country_code_opt = None; assert_eq!(result_node_record_inner, our_node_record.inner); let our_cryptde = CryptDENull::from(our_node_record.public_key(), TEST_DEFAULT_CHAIN); assert_eq!( diff --git a/node/src/neighborhood/mod.rs b/node/src/neighborhood/mod.rs index e46e2abce..d81190ff4 100644 --- a/node/src/neighborhood/mod.rs +++ b/node/src/neighborhood/mod.rs @@ -5,43 +5,25 @@ pub mod gossip; pub mod gossip_acceptor; pub mod gossip_producer; pub mod neighborhood_database; +pub mod node_location; pub mod node_record; pub mod overall_connection_status; -use std::collections::HashSet; -use std::convert::TryFrom; -use std::net::{IpAddr, SocketAddr}; -use std::path::PathBuf; - -use actix::Context; -use actix::Handler; -use actix::MessageResult; -use actix::Recipient; -use actix::{Actor, System}; -use actix::{Addr, AsyncContext}; -use itertools::Itertools; -use masq_lib::messages::{ - FromMessageBody, ToMessageBody, UiConnectionStage, UiConnectionStatusRequest, -}; -use masq_lib::messages::{UiConnectionStatusResponse, UiShutdownRequest}; -use masq_lib::ui_gateway::{MessageTarget, NodeFromUiMessage, NodeToUiMessage}; -use masq_lib::utils::{exit_process, ExpectValue, NeighborhoodModeLight}; - use crate::bootstrapper::BootstrapperConfig; use crate::database::db_initializer::DbInitializationConfig; use crate::database::db_initializer::{DbInitializer, DbInitializerReal}; use crate::db_config::persistent_configuration::{ PersistentConfigError, PersistentConfiguration, PersistentConfigurationReal, }; -use crate::neighborhood::gossip::{DotGossipEndpoint, GossipNodeRecord, Gossip_0v1}; +use crate::neighborhood::gossip::{AccessibleGossipRecord, DotGossipEndpoint, Gossip_0v1}; use crate::neighborhood::gossip_acceptor::GossipAcceptanceResult; -use crate::neighborhood::node_record::NodeRecordInner_0v1; +use crate::neighborhood::node_location::get_node_location; use crate::neighborhood::overall_connection_status::{ OverallConnectionStage, OverallConnectionStatus, }; use crate::stream_messages::RemovedStreamType; +use crate::sub_lib::cryptde::CryptDE; use crate::sub_lib::cryptde::PublicKey; -use crate::sub_lib::cryptde::{CryptDE, CryptData, PlainData}; use crate::sub_lib::dispatcher::{Component, StreamShutdownMsg}; use crate::sub_lib::hopper::{ExpiredCoresPackage, NoLookupIncipientCoresPackage}; use crate::sub_lib::hopper::{IncipientCoresPackage, MessageType}; @@ -66,20 +48,49 @@ use crate::sub_lib::utils::{ }; use crate::sub_lib::versioned_data::VersionedData; use crate::sub_lib::wallet::Wallet; +use actix::Context; +use actix::Handler; +use actix::MessageResult; +use actix::Recipient; +use actix::{Actor, System}; +use actix::{Addr, AsyncContext}; use gossip_acceptor::GossipAcceptor; use gossip_acceptor::GossipAcceptorReal; use gossip_producer::GossipProducer; use gossip_producer::GossipProducerReal; +use itertools::Itertools; use masq_lib::blockchains::chains::Chain; +use masq_lib::constants::{ + DEFAULT_PREALLOCATION_VEC, EXIT_COUNTRY_MISSING_COUNTRIES_ERROR, PAYLOAD_ZERO_SIZE, +}; use masq_lib::crash_point::CrashPoint; +use masq_lib::exit_locations::ExitLocationSet; use masq_lib::logger::Logger; +use masq_lib::messages::{ + ExitLocation, FromMessageBody, ToMessageBody, UiConnectionStage, UiConnectionStatusRequest, + UiSetExitLocationRequest, UiSetExitLocationResponse, +}; +use masq_lib::messages::{UiConnectionStatusResponse, UiShutdownRequest}; +use masq_lib::ui_gateway::MessagePath::Conversation; +use masq_lib::ui_gateway::{MessageBody, MessageTarget, NodeFromUiMessage, NodeToUiMessage}; +use masq_lib::utils::{exit_process, ExpectValue, NeighborhoodModeLight}; use neighborhood_database::NeighborhoodDatabase; use node_record::NodeRecord; +use std::collections::HashSet; +use std::convert::TryFrom; +use std::fmt::Debug; +use std::net::{IpAddr, SocketAddr}; +use std::path::PathBuf; +use std::string::ToString; pub const CRASH_KEY: &str = "NEIGHBORHOOD"; pub const DEFAULT_MIN_HOPS: Hops = Hops::ThreeHops; pub const UNREACHABLE_HOST_PENALTY: i64 = 100_000_000; +pub const UNREACHABLE_COUNTRY_PENALTY: u32 = 100_000_000; +pub const ZERO_UNDESIRABILITY: u32 = 0; +pub const COUNTRY_UNDESIRABILITY_FACTOR: u32 = 1_000; pub const RESPONSE_UNDESIRABILITY_FACTOR: usize = 1_000; // assumed response length is request * this +pub const ZZ_COUNTRY_CODE_STRING: &str = "ZZ"; pub struct Neighborhood { cryptde: &'static dyn CryptDE, @@ -103,6 +114,7 @@ pub struct Neighborhood { db_password_opt: Option, logger: Logger, tools: NeighborhoodTools, + user_exit_preferences: UserExitPreferences, } impl Actor for Neighborhood { @@ -114,10 +126,7 @@ impl Handler for Neighborhood { fn handle(&mut self, msg: BindMessage, ctx: &mut Self::Context) -> Self::Result { ctx.set_mailbox_capacity(NODE_MAILBOX_CAPACITY); - self.hopper_opt = Some(msg.peer_actors.hopper.from_hopper_client); - self.hopper_no_lookup_opt = Some(msg.peer_actors.hopper.from_hopper_client_no_lookup); - self.connected_signal_opt = Some(msg.peer_actors.accountant.start); - self.node_to_ui_recipient_opt = Some(msg.peer_actors.ui_gateway.node_to_ui_message_sub); + self.handle_bind_message(msg); } } @@ -352,7 +361,9 @@ impl Handler for Neighborhood { fn handle(&mut self, msg: NodeFromUiMessage, _ctx: &mut Self::Context) -> Self::Result { let client_id = msg.client_id; - if let Ok((_, context_id)) = UiConnectionStatusRequest::fmb(msg.body.clone()) { + if let Ok((message, context_id)) = UiSetExitLocationRequest::fmb(msg.body.clone()) { + self.handle_exit_location_message(message, client_id, context_id); + } else if let Ok((_, context_id)) = UiConnectionStatusRequest::fmb(msg.body.clone()) { self.handle_connection_status_message(client_id, context_id); } else if let Ok((body, _)) = UiShutdownRequest::fmb(msg.body.clone()) { self.handle_shutdown_order(client_id, body); @@ -362,38 +373,6 @@ impl Handler for Neighborhood { } } -#[derive(Debug, PartialEq, Eq, Clone)] -pub struct AccessibleGossipRecord { - pub signed_gossip: PlainData, - pub signature: CryptData, - pub node_addr_opt: Option, - pub inner: NodeRecordInner_0v1, -} - -impl AccessibleGossipRecord { - pub fn regenerate_signed_gossip(&mut self, cryptde: &dyn CryptDE) { - let (signed_gossip, signature) = regenerate_signed_gossip(&self.inner, cryptde); - self.signed_gossip = signed_gossip; - self.signature = signature; - } -} - -impl TryFrom for AccessibleGossipRecord { - type Error = String; - - fn try_from(value: GossipNodeRecord) -> Result { - match serde_cbor::de::from_slice(value.signed_data.as_slice()) { - Ok(inner) => Ok(AccessibleGossipRecord { - signed_gossip: value.signed_data, - signature: value.signature, - node_addr_opt: value.node_addr_opt, - inner, - }), - Err(e) => Err(format!("{}", e)), - } - } -} - #[derive(Debug, PartialEq, Eq, Clone, Copy)] enum RouteDirection { Over, @@ -459,6 +438,7 @@ impl Neighborhood { db_password_opt: config.db_password_opt.clone(), logger: Logger::new("Neighborhood"), tools: NeighborhoodTools::default(), + user_exit_preferences: UserExitPreferences::new(), } } @@ -500,12 +480,20 @@ impl Neighborhood { .expectv("Root node") .ip_addr(); self.neighborhood_database.new_public_ip(new_public_ip); + self.handle_new_ip_location(new_public_ip); info!( self.logger, "Changed public IP from {} to {}", old_public_ip, new_public_ip ); } + fn handle_new_ip_location(&mut self, new_public_ip: IpAddr) { + let node_location_opt = get_node_location(Some(new_public_ip)); + let root_node = self.neighborhood_database.root_mut(); + root_node.metadata.node_location_opt = node_location_opt.clone(); + root_node.inner.country_code_opt = node_location_opt.map(|nl| nl.country_code); + } + fn handle_route_query_message(&mut self, msg: RouteQueryMessage) -> Option { let debug_msg_opt = self.logger.debug_enabled().then(|| format!("{:?}", msg)); let route_result = if self.mode == NeighborhoodModeLight::ZeroHop { @@ -568,6 +556,15 @@ impl Neighborhood { &self.logger, ); } + self.user_exit_preferences.db_countries = self.init_db_countries(); + if let Some(exit_locations_by_priority) = + self.user_exit_preferences.locations_opt.clone() + { + for exit_location in &exit_locations_by_priority { + self.enrich_exit_countries(&exit_location.country_codes); + } + self.set_country_undesirability_and_exit_countries(&exit_locations_by_priority); + } self.search_for_a_new_route(); } ConfigChange::UpdatePassword(new_password) => { @@ -779,6 +776,7 @@ impl Neighborhood { connection_progress_peers: self.overall_connection_status.get_peer_addrs(), cpm_recipient, db_patch_size: self.db_patch_size, + user_exit_preferences_opt: Some(self.user_exit_preferences.clone()), }; let acceptance_result = self.gossip_acceptor.handle( &mut self.neighborhood_database, @@ -787,8 +785,15 @@ impl Neighborhood { neighborhood_metadata, ); match acceptance_result { - GossipAcceptanceResult::Accepted => self.gossip_to_neighbors(), + GossipAcceptanceResult::Accepted => { + self.user_exit_preferences.db_countries = self.init_db_countries(); + self.gossip_to_neighbors() + } GossipAcceptanceResult::Reply(next_debut, target_key, target_node_addr) => { + //TODO also ensure init_db_countries on hop change + if self.min_hops == Hops::OneHop { + self.user_exit_preferences.db_countries = self.init_db_countries(); + } self.handle_gossip_reply(next_debut, &target_key, &target_node_addr) } GossipAcceptanceResult::Failed(failure, target_key, target_node_addr) => { @@ -799,8 +804,9 @@ impl Neighborhood { self.handle_gossip_ignored(ignored_node_name, gossip_record_count) } GossipAcceptanceResult::Ban(reason) => { - warning!(self.logger, "Malefactor detected at {}, but malefactor bans not yet implemented; ignoring: {}", gossip_source, reason - ); + // TODO in case we introduce Ban machinery we want to reinitialize the db_countries here as well + // in that case, we need to make new process in init_db_countries to exclude banned node, from the result + warning!(self.logger, "Malefactor detected at {}, but malefactor bans not yet implemented; ignoring: {}", gossip_source, reason); self.handle_gossip_ignored(ignored_node_name, gossip_record_count); } } @@ -1191,38 +1197,108 @@ impl Neighborhood { } } + fn validate_country_code_when_fallback_routing(&self, last_node: &PublicKey) -> bool { + let last_cc = match self.neighborhood_database.node_by_key(last_node) { + Some(nr) => nr + .inner + .country_code_opt + .clone() + .unwrap_or_else(|| "ZZ".to_string()), + None => "ZZ".to_string(), + }; + if self.user_exit_preferences.exit_countries.contains(&last_cc) { + return true; + } + if self.user_exit_preferences.exit_countries.is_empty() { + return true; + } + for country in &self.user_exit_preferences.exit_countries { + if self.user_exit_preferences.db_countries.contains(country) && country != &last_cc { + return false; + } + } + true + } + + fn validate_last_node_country_code( + &self, + last_node_key: &PublicKey, + research_neighborhood: bool, + direction: RouteDirection, + ) -> bool { + if self.is_always_true(last_node_key, research_neighborhood, direction) { + true // Zero- and single-hop routes are not subject to exit-too-close restrictions + } else { + if let Some(node_record) = self.neighborhood_database.node_by_key(last_node_key) { + if let Some(country_code) = &node_record.inner.country_code_opt { + return self + .user_exit_preferences + .exit_countries + .contains(country_code); + } + } + false + } + } + + fn is_always_true( + &self, + last_node_key: &PublicKey, + research_neighborhood: bool, + direction: RouteDirection, + ) -> bool { + self.user_exit_preferences.fallback_preference == FallbackPreference::Nothing + || (self.user_exit_preferences.fallback_preference + == FallbackPreference::ExitCountryWithFallback + && self.validate_country_code_when_fallback_routing(last_node_key)) + || research_neighborhood + || direction == RouteDirection::Back + } + fn compute_undesirability( node_record: &NodeRecord, payload_size: u64, undesirability_type: UndesirabilityType, logger: &Logger, ) -> i64 { - let mut rate_undesirability = match undesirability_type { - UndesirabilityType::Relay => node_record.inner.rate_pack.routing_charge(payload_size), - UndesirabilityType::ExitRequest(_) => { - node_record.inner.rate_pack.exit_charge(payload_size) + match undesirability_type { + UndesirabilityType::Relay => { + node_record.inner.rate_pack.routing_charge(payload_size) as i64 } - UndesirabilityType::ExitAndRouteResponse => { - node_record.inner.rate_pack.exit_charge(payload_size) - + node_record.inner.rate_pack.routing_charge(payload_size) + UndesirabilityType::ExitRequest(None) => { + node_record.inner.rate_pack.exit_charge(payload_size) as i64 + + node_record.metadata.country_undesirability as i64 } - } as i64; - if let UndesirabilityType::ExitRequest(Some(hostname)) = undesirability_type { - if node_record.metadata.unreachable_hosts.contains(hostname) { - trace!( - logger, - "Node with PubKey {:?} failed to reach host {:?} during ExitRequest; Undesirability: {} + {} = {}", - node_record.public_key(), - hostname, - rate_undesirability, - UNREACHABLE_HOST_PENALTY, - rate_undesirability + UNREACHABLE_HOST_PENALTY - ); - rate_undesirability += UNREACHABLE_HOST_PENALTY; + UndesirabilityType::ExitRequest(Some(hostname)) => { + let exit_undesirability = + node_record.inner.rate_pack.exit_charge(payload_size) as i64; + let country_undesirability = node_record.metadata.country_undesirability as i64; + let unreachable_undesirability = if node_record + .metadata + .unreachable_hosts + .contains(hostname) + { + trace!( + logger, + "Node with PubKey {:?} failed to reach host {:?} during ExitRequest; Undesirability: {} + {} + {} = {}", + node_record.public_key(), + hostname, + exit_undesirability, + UNREACHABLE_HOST_PENALTY, + country_undesirability, + exit_undesirability + UNREACHABLE_HOST_PENALTY + country_undesirability + ); + UNREACHABLE_HOST_PENALTY + } else { + 0i64 + }; + exit_undesirability + unreachable_undesirability + country_undesirability + } + UndesirabilityType::ExitAndRouteResponse => { + node_record.inner.rate_pack.exit_charge(payload_size) as i64 + + node_record.inner.rate_pack.routing_charge(payload_size) as i64 } } - - rate_undesirability } fn is_orig_node_on_back_leg( @@ -1245,6 +1321,31 @@ impl Neighborhood { return_route_id } + pub fn find_exit_locations<'a>( + &'a self, + source: &'a PublicKey, + minimum_hops: usize, + ) -> Vec<&'a PublicKey> { + let mut minimum_undesirability = i64::MAX; + let initial_undesirability = 0; + let research_exits: &mut Vec<&'a PublicKey> = &mut vec![]; + let mut prefix = Vec::with_capacity(DEFAULT_PREALLOCATION_VEC); + prefix.push(source); + let _ = self.routing_engine( + prefix, + initial_undesirability, + None, + minimum_hops, + PAYLOAD_ZERO_SIZE, + RouteDirection::Over, + &mut minimum_undesirability, + None, + true, + research_exits, + ); + research_exits.to_vec() + } + // Interface to main routing engine. Supply source key, target key--if any--in target_opt, // minimum hops, size of payload in bytes, the route direction, and the hostname if you know it. // @@ -1252,6 +1353,7 @@ impl Neighborhood { // target in hops_remaining or more hops with no cycles, or from the origin hops_remaining hops // out into the MASQ Network. No round trips; if you want a round trip, call this method twice. // If the return value is None, no qualifying route was found. + #[allow(clippy::too_many_arguments)] fn find_best_route_segment<'a>( &'a self, source: &'a PublicKey, @@ -1264,6 +1366,9 @@ impl Neighborhood { let mut minimum_undesirability = i64::MAX; let initial_undesirability = self.compute_initial_undesirability(source, payload_size as u64, direction); + let mut prefix = Vec::with_capacity(DEFAULT_PREALLOCATION_VEC); + //TODO we can have an investigation, if this DEFAULT_PREALLOCATION_VEC is not too much, same in find_exit_locations + prefix.push(source); let result = self .routing_engine( vec![source], @@ -1274,6 +1379,8 @@ impl Neighborhood { direction, &mut minimum_undesirability, hostname_opt, + false, + &mut vec![], ) .into_iter() .filter_map(|cr| match cr.undesirability <= minimum_undesirability { @@ -1296,8 +1403,10 @@ impl Neighborhood { direction: RouteDirection, minimum_undesirability: &mut i64, hostname_opt: Option<&str>, + research_neighborhood: bool, + research_exits: &mut Vec<&'a PublicKey>, ) -> Vec> { - if undesirability > *minimum_undesirability { + if undesirability > *minimum_undesirability && !research_neighborhood { return vec![]; } let first_node_key = prefix.first().expect("Empty prefix"); @@ -1314,58 +1423,121 @@ impl Neighborhood { previous_node.public_key(), ) { - if undesirability < *minimum_undesirability { - *minimum_undesirability = undesirability; + if !research_neighborhood + && self.validate_last_node_country_code( + previous_node.public_key(), + research_neighborhood, + direction, + ) + { + if undesirability < *minimum_undesirability { + *minimum_undesirability = undesirability; + } + vec![ComputedRouteSegment::new(prefix.clone(), undesirability)] + } else if research_neighborhood && research_exits.contains(&prefix[prefix.len() - 1]) { + vec![] + } else { + if research_neighborhood { + research_exits.push(prefix[prefix.len() - 1]); + } + self.routing_guts( + prefix, + undesirability, + target_opt, + hops_remaining, + payload_size, + direction, + minimum_undesirability, + hostname_opt, + research_neighborhood, + research_exits, + previous_node, + ) } - vec![ComputedRouteSegment::new(prefix, undesirability)] - } else if (hops_remaining == 0) && target_opt.is_none() { + } else if ((hops_remaining == 0) && target_opt.is_none() && !research_neighborhood) + && (self.user_exit_preferences.fallback_preference == FallbackPreference::Nothing + || self.user_exit_preferences.exit_countries.is_empty()) + { + // in case we do not investigate neighborhood for country codes, or we are not looking for particular country exit: // don't continue a targetless search past the minimum hop count vec![] } else { - // Go through all the neighbors and compute shorter routes through all the ones we're not already using. - previous_node - .full_neighbors(&self.neighborhood_database) - .iter() - .filter(|node_record| !prefix.contains(&node_record.public_key())) - .filter(|node_record| { - node_record.routes_data() - || Self::is_orig_node_on_back_leg(**node_record, target_opt, direction) - }) - .flat_map(|node_record| { - let mut new_prefix = prefix.clone(); - new_prefix.push(node_record.public_key()); - - let new_hops_remaining = if hops_remaining == 0 { - 0 - } else { - hops_remaining - 1 - }; - - let new_undesirability = self.compute_new_undesirability( - node_record, - undesirability, - target_opt, - new_hops_remaining, - payload_size as u64, - direction, - hostname_opt, - ); - - self.routing_engine( - new_prefix.clone(), - new_undesirability, - target_opt, - new_hops_remaining, - payload_size, - direction, - minimum_undesirability, - hostname_opt, - ) - }) - .collect() + self.routing_guts( + prefix, + undesirability, + target_opt, + hops_remaining, + payload_size, + direction, + minimum_undesirability, + hostname_opt, + research_neighborhood, + research_exits, + previous_node, + ) } } + #[allow(clippy::too_many_arguments)] + fn routing_guts<'a>( + &'a self, + prefix: Vec<&'a PublicKey>, + undesirability: i64, + target_opt: Option<&'a PublicKey>, + hops_remaining: usize, + payload_size: usize, + direction: RouteDirection, + minimum_undesirability: &mut i64, + hostname_opt: Option<&str>, + research_neighborhood: bool, + exits_research: &mut Vec<&'a PublicKey>, + previous_node: &NodeRecord, + ) -> Vec { + // Go through all the neighbors and compute shorter routes through all the ones we're not already using. + previous_node + .full_neighbors(&self.neighborhood_database) + .iter() + .filter(|node_record| !prefix.contains(&node_record.public_key())) + .filter(|node_record| { + node_record.routes_data() + || Self::is_orig_node_on_back_leg(**node_record, target_opt, direction) + }) + .flat_map(|node_record| { + let mut new_prefix = prefix.clone(); + new_prefix.push(node_record.public_key()); + + let new_hops_remaining = if hops_remaining == 0 { + 0 + } else { + hops_remaining - 1 + }; + + let new_undesirability = self.compute_new_undesirability( + node_record, + undesirability, + target_opt, + new_hops_remaining, + payload_size as u64, + direction, + hostname_opt, + ); + + self.routing_engine( + new_prefix, + new_undesirability, + target_opt, + new_hops_remaining, + payload_size, + direction, + minimum_undesirability, + hostname_opt, + research_neighborhood, + exits_research, + ) + }) + .collect() + } + fn send_ask_about_debut_gossip_message( &mut self, ctx: &mut Context, @@ -1434,6 +1606,279 @@ impl Neighborhood { undesirability + node_undesirability } + fn handle_exit_location_message( + &mut self, + message: UiSetExitLocationRequest, + client_id: u64, + context_id: u64, + ) { + //TODO write test that contains more CountryGroups than countries in neighborhood db to check if unexistent country codes in db are filtered out from ExitLocation + let (exit_locations_by_priority, missing_countries) = + self.extract_exit_locations_from_message(&message); + + self.user_exit_preferences.fallback_preference = match ( + message.fallback_routing, + exit_locations_by_priority.is_empty(), + ) { + (true, true) | (false, true) => FallbackPreference::Nothing, + (true, false) => FallbackPreference::ExitCountryWithFallback, + (false, false) => FallbackPreference::ExitCountryNoFallback, + }; + + let fallback_status = match self.user_exit_preferences.fallback_preference { + FallbackPreference::Nothing | FallbackPreference::ExitCountryWithFallback => { + "Fallback Routing is set." + } + FallbackPreference::ExitCountryNoFallback => "Fallback Routing NOT set.", + }; + + if !message.show_countries { + self.set_exit_locations_opt(&exit_locations_by_priority); + } + match self.neighborhood_database.keys().len() > 1 { + true => { + self.set_country_undesirability_and_exit_countries(&exit_locations_by_priority); + self.exit_location_logger_output( + exit_locations_by_priority, + &missing_countries, + fallback_status, + ); + } + false => info!( + self.logger, + "Neighborhood is empty, no exit Nodes are available.", + ), + } + let message = self.create_exit_location_response( + client_id, + context_id, + missing_countries, + message.show_countries, + ); + self.node_to_ui_recipient_opt + .as_ref() + .expect("UI Gateway is unbound") + .try_send(message) + .expect("UiGateway is dead"); + } + + fn exit_location_logger_output( + &mut self, + exit_locations_by_priority: Vec, + missing_locations: &Vec, + fallback_status: &str, + ) { + self.logger.info(|| { + let location_set = ExitLocationSet { + locations: exit_locations_by_priority, + }; + let exit_location_status = match location_set.locations.is_empty() { + false => "Exit location set: ", + true => "Exit location unset.", + }; + format!( + "{} {}{}", + fallback_status, exit_location_status, location_set + ) + }); + if !missing_locations.is_empty() { + warning!( + self.logger, + "Exit Location: following desired countries are missing in Neighborhood {:?}", + &missing_locations + ); + } + } + + fn error_message_indicates(&self, missing_countries: &mut Vec) -> bool { + let mut desired_countries: Vec = vec![]; + if let Some(exit_vec) = self.user_exit_preferences.locations_opt.as_ref() { + for location in exit_vec { + let mut to_append = location.country_codes.clone(); + desired_countries.append(&mut to_append) + } + } + if desired_countries.is_empty() && missing_countries.is_empty() { + return false; + } + desired_countries.sort(); + missing_countries.sort(); + missing_countries == &desired_countries + } + + fn create_exit_location_response( + &self, + client_id: u64, + context_id: u64, + mut missing_countries: Vec, + show_countries_flag: bool, + ) -> NodeToUiMessage { + let fallback_routing = self.is_fallback_routing_active(); + let exit_locations = self.get_locations_opt(); + let countries_to_show = self.get_countries_to_show(show_countries_flag); + let missing_countries_message: String = missing_countries.join(", "); + if self.error_message_indicates(&mut missing_countries) { + NodeToUiMessage { + target: MessageTarget::ClientId(client_id), + body: MessageBody { + opcode: "exitLocation".to_string(), + path: Conversation(context_id), + payload: Err(( + EXIT_COUNTRY_MISSING_COUNTRIES_ERROR, + missing_countries_message, + )), + }, + } + } else { + NodeToUiMessage { + target: MessageTarget::ClientId(client_id), + body: UiSetExitLocationResponse { + fallback_routing, + exit_country_selection: exit_locations, + exit_countries: countries_to_show, + missing_countries, + } + .tmb(context_id), + } + } + } + + fn get_countries_to_show(&self, show_countries_flag: bool) -> Option> { + match show_countries_flag { + true => Some(self.user_exit_preferences.db_countries.clone()), + false => None, + } + } + + fn is_fallback_routing_active(&self) -> bool { + match &self.user_exit_preferences.fallback_preference { + FallbackPreference::Nothing => true, + FallbackPreference::ExitCountryWithFallback => true, + FallbackPreference::ExitCountryNoFallback => false, + } + } + + fn get_locations_opt(&self) -> Vec { + self.user_exit_preferences + .locations_opt + .clone() + .unwrap_or_default() + } + + fn set_exit_locations_opt(&mut self, exit_locations_by_priority: &[ExitLocation]) { + self.user_exit_preferences.locations_opt = + match self.user_exit_preferences.exit_countries.is_empty() { + false => Some(exit_locations_by_priority.to_owned()), + true => match self.user_exit_preferences.fallback_preference { + FallbackPreference::ExitCountryNoFallback => None, + _ => Some(exit_locations_by_priority.to_owned()), + }, + }; + } + + fn set_country_undesirability_and_exit_countries( + &mut self, + exit_locations_by_priority: &Vec, + ) { + let nodes = self.neighborhood_database.nodes_mut(); + match !&exit_locations_by_priority.is_empty() { + true => { + for node_record in nodes { + self.user_exit_preferences + .assign_nodes_country_undesirability(node_record) + } + } + false => { + self.user_exit_preferences.exit_countries = vec![]; + for node_record in nodes { + node_record.metadata.country_undesirability = ZERO_UNDESIRABILITY; + } + } + } + } + + fn extract_exit_locations_from_message( + &mut self, + message: &UiSetExitLocationRequest, + ) -> (Vec, Vec) { + self.user_exit_preferences.db_countries = self.init_db_countries(); + let mut countries_not_in_neighborhood = vec![]; + ( + message + .to_owned() + .exit_locations + .into_iter() + .map(|cc| { + let requested_country_codes = &cc.country_codes; + countries_not_in_neighborhood + .extend(self.enrich_exit_countries(requested_country_codes)); + ExitLocation { + country_codes: cc.country_codes, + priority: cc.priority, + } + }) + .collect(), + countries_not_in_neighborhood, + ) + } + + fn enrich_exit_countries(&mut self, country_codes: &Vec) -> Vec { + let mut countries_not_in_neighborhood = vec![]; + for code in country_codes { + if self.code_in_db_countries_or_fallback_active(code) { + if !self.user_exit_preferences.exit_countries.contains(code) { + self.user_exit_preferences.exit_countries.push(code.clone()); + } + if self.fallback_active_and_code_missing_in_db_countries(code) { + countries_not_in_neighborhood.push(code.clone()); + } + } else { + if let Some(index) = self + .user_exit_preferences + .exit_countries + .iter() + .position(|item| item.eq(code)) + { + self.user_exit_preferences.exit_countries.remove(index); + } + countries_not_in_neighborhood.push(code.clone()); + } + } + countries_not_in_neighborhood + } + + fn fallback_active_and_code_missing_in_db_countries(&mut self, code: &String) -> bool { + (self.user_exit_preferences.fallback_preference + == FallbackPreference::ExitCountryWithFallback) + && !self.user_exit_preferences.db_countries.contains(code) + } + + fn code_in_db_countries_or_fallback_active(&mut self, code: &String) -> bool { + self.user_exit_preferences.db_countries.contains(code) + || (self.user_exit_preferences.fallback_preference + == FallbackPreference::ExitCountryWithFallback) + } + + fn init_db_countries(&mut self) -> Vec { + let root_key = self.neighborhood_database.root_key(); + let min_hops = self.min_hops as usize; + let exit_nodes = self.find_exit_locations(root_key, min_hops).to_owned(); + let mut db_countries = vec![]; + if !exit_nodes.is_empty() { + for pub_key in exit_nodes { + let node_opt = self.neighborhood_database.node_by_key(pub_key); + if let Some(node_record) = node_opt { + if let Some(cc) = &node_record.inner.country_code_opt { + db_countries.push(cc.clone()) + } + } + } + } + db_countries.sort(); + db_countries.dedup(); + db_countries + } + fn handle_gossip_reply( &self, gossip: Gossip_0v1, @@ -1595,19 +2040,84 @@ impl Neighborhood { self.db_patch_size = Neighborhood::calculate_db_patch_size(new_min_hops); debug!(self.logger, "The value of min_hops ({}-hop -> {}-hop) and db_patch_size ({} -> {}) has been changed", prev_min_hops, self.min_hops, prev_db_patch_size, self.db_patch_size); } -} -pub fn regenerate_signed_gossip( - inner: &NodeRecordInner_0v1, - cryptde: &dyn CryptDE, // Must be the correct CryptDE for the Node from which inner came: used for signing -) -> (PlainData, CryptData) { - let signed_gossip = - PlainData::from(serde_cbor::ser::to_vec(&inner).expect("Serialization failed")); - let signature = match cryptde.sign(&signed_gossip) { - Ok(sig) => sig, - Err(e) => unimplemented!("TODO: Signing error: {:?}", e), - }; - (signed_gossip, signature) + fn handle_bind_message(&mut self, msg: BindMessage) { + self.hopper_opt = Some(msg.peer_actors.hopper.from_hopper_client); + self.hopper_no_lookup_opt = Some(msg.peer_actors.hopper.from_hopper_client_no_lookup); + self.connected_signal_opt = Some(msg.peer_actors.accountant.start); + self.node_to_ui_recipient_opt = Some(msg.peer_actors.ui_gateway.node_to_ui_message_sub); + } +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct ExitLocationsRoutes<'a> { + routes: Vec<(Vec<&'a PublicKey>, i64)>, +} + +#[derive(Debug, PartialEq, Eq, Clone)] +pub enum FallbackPreference { + Nothing, + ExitCountryWithFallback, + ExitCountryNoFallback, +} + +//TODO create big comment about all members and its utilization +#[derive(Clone, Debug)] +pub struct UserExitPreferences { + exit_countries: Vec, //if we cross number of country_codes used in one workflow over 34, we want to change this member to HashSet + fallback_preference: FallbackPreference, + locations_opt: Option>, //TODO remove Option from NeighborhoodMetadata and create there TODO to optimize it in future via reference + db_countries: Vec, +} + +impl UserExitPreferences { + fn new() -> UserExitPreferences { + UserExitPreferences { + exit_countries: vec![], + fallback_preference: FallbackPreference::Nothing, + locations_opt: None, + db_countries: vec![], + } + } + + pub fn assign_nodes_country_undesirability(&self, node_record: &mut NodeRecord) { + let country_code = node_record + .inner + .country_code_opt + .clone() + .unwrap_or_else(|| ZZ_COUNTRY_CODE_STRING.to_string()); + match &self.locations_opt { + Some(exit_locations_by_priority) => { + for exit_location in exit_locations_by_priority { + if Self::should_set_country_undesirability(&country_code, exit_location) { + node_record.metadata.country_undesirability = + Self::calculate_country_undesirability(exit_location.priority as u32); + } + if self.is_unreachable_country_penalty(&country_code) { + node_record.metadata.country_undesirability = UNREACHABLE_COUNTRY_PENALTY; + } + } + } + None => (), + } + } + + fn should_set_country_undesirability( + country_code: &String, + exit_location: &ExitLocation, + ) -> bool { + exit_location.country_codes.contains(country_code) && country_code != ZZ_COUNTRY_CODE_STRING + } + + fn is_unreachable_country_penalty(&self, country_code: &String) -> bool { + (self.fallback_preference == FallbackPreference::ExitCountryWithFallback + && !self.exit_countries.contains(country_code)) + || country_code == ZZ_COUNTRY_CODE_STRING + } + + fn calculate_country_undesirability(priority: u32) -> u32 { + COUNTRY_UNDESIRABILITY_FACTOR * (priority - 1u32) + } } #[derive(PartialEq, Eq, Debug)] @@ -1617,6 +2127,7 @@ enum UndesirabilityType<'hostname> { ExitAndRouteResponse, } +#[derive(Debug)] struct ComputedRouteSegment<'a> { pub nodes: Vec<&'a PublicKey>, pub undesirability: i64, @@ -1639,6 +2150,7 @@ mod tests { use serde_cbor; use std::any::TypeId; use std::cell::RefCell; + use std::collections::HashMap; use std::convert::TryInto; use std::net::{IpAddr, SocketAddr}; use std::path::Path; @@ -1650,19 +2162,21 @@ mod tests { use tokio::prelude::Future; use masq_lib::constants::{DEFAULT_CHAIN, TLS_PORT}; - use masq_lib::messages::{ToMessageBody, UiConnectionChangeBroadcast, UiConnectionStage}; + use masq_lib::messages::{ + CountryGroups, ToMessageBody, UiConnectionChangeBroadcast, UiConnectionStage, + }; use masq_lib::test_utils::utils::{ensure_node_home_directory_exists, TEST_DEFAULT_CHAIN}; - use masq_lib::ui_gateway::MessageBody; use masq_lib::ui_gateway::MessagePath::Conversation; use masq_lib::ui_gateway::MessageTarget; + use masq_lib::ui_gateway::{MessageBody, MessagePath}; use masq_lib::utils::running_test; use crate::db_config::persistent_configuration::PersistentConfigError; - use crate::neighborhood::gossip::GossipBuilder; use crate::neighborhood::gossip::Gossip_0v1; - use crate::neighborhood::node_record::NodeRecordInner_0v1; + use crate::neighborhood::gossip::{GossipBuilder, GossipNodeRecord}; + use crate::neighborhood::node_record::{NodeRecordInner_0v1, NodeRecordInputs}; use crate::stream_messages::{NonClandestineAttributes, RemovedStreamType}; - use crate::sub_lib::cryptde::{decodex, encodex, CryptData}; + use crate::sub_lib::cryptde::{decodex, encodex, CryptData, PlainData}; use crate::sub_lib::cryptde_null::CryptDENull; use crate::sub_lib::dispatcher::Endpoint; use crate::sub_lib::hop::LiveHop; @@ -1710,6 +2224,22 @@ mod tests { use crate::test_utils::unshared_test_utils::notify_handlers::NotifyLaterHandleMock; use masq_lib::test_utils::logging::{init_test_logging, TestLogHandler}; + impl Neighborhood { + fn get_node_country_undesirability(&self, pubkey: &PublicKey) -> u32 { + self.neighborhood_database + .node_by_key(pubkey) + .unwrap() + .metadata + .country_undesirability + } + } + + impl NeighborhoodDatabase { + pub fn set_root_key(&mut self, key: &PublicKey) { + self.this_node = key.clone(); + } + } + impl Handler> for Neighborhood { type Result = (); @@ -1753,6 +2283,134 @@ mod tests { assert_eq!(subject.db_patch_size, expected_db_patch_size); } + #[test] + fn init_db_countries_works_properly() { + let mut subject = make_standard_subject(); + subject.min_hops = Hops::OneHop; + let root_node_key = subject.neighborhood_database.root().public_key().clone(); + let mut first_neighbor = make_node_record(1111, true); + first_neighbor.inner.country_code_opt = Some("CZ".to_string()); + let mut second_neighbor = make_node_record(2222, true); + second_neighbor.inner.country_code_opt = Some("DE".to_string()); + subject + .neighborhood_database + .add_node(first_neighbor.clone()) + .unwrap(); + subject + .neighborhood_database + .add_node(second_neighbor.clone()) + .unwrap(); + subject + .neighborhood_database + .add_arbitrary_full_neighbor(&root_node_key, first_neighbor.public_key()); + subject + .neighborhood_database + .add_arbitrary_full_neighbor(&root_node_key, second_neighbor.public_key()); + let filled_db_countries = subject.init_db_countries(); + + subject + .neighborhood_database + .remove_arbitrary_half_neighbor(&root_node_key, first_neighbor.public_key()); + subject + .neighborhood_database + .remove_arbitrary_half_neighbor(&root_node_key, second_neighbor.public_key()); + let emptied_db_countries = subject.init_db_countries(); + + assert_eq!(filled_db_countries, &["CZ".to_string(), "DE".to_string()]); + assert!(emptied_db_countries.is_empty()); + } + + #[test] + fn standard_gossip_results_in_exit_node_in_database() { + let mut subject = make_standard_subject(); + let root_node_key = subject.neighborhood_database.root_key().clone(); + let mut source_node = make_node_record(1111, true); //US + source_node.inner.country_code_opt = Some("US".to_string()); + let mut first_node = make_node_record(2222, true); //FR + first_node.inner.country_code_opt = Some("FR".to_string()); + let second_node = make_node_record(3333, false); + subject + .neighborhood_database + .add_node(source_node.clone()) + .unwrap(); + subject + .neighborhood_database + .add_node(second_node.clone()) + .unwrap(); + subject + .neighborhood_database + .add_arbitrary_full_neighbor(&root_node_key, source_node.public_key()); + let mut source_db = subject.neighborhood_database.clone(); + source_db.set_root_key(source_node.public_key()); + source_db.add_arbitrary_full_neighbor(source_node.public_key(), &root_node_key); + source_db.add_node(first_node.clone()).unwrap(); + source_db.add_arbitrary_full_neighbor(source_node.public_key(), first_node.public_key()); + source_db.root_mut().inner.version = 1; + let resigner = source_db.node_by_key_mut(source_node.public_key()).unwrap(); + resigner.resign(); + let standard_gossip = GossipBuilder::new(&source_db) + .node(source_node.public_key(), true) + .node(second_node.public_key(), false) + .node(first_node.public_key(), false) + .build(); + let peer_actors = peer_actors_builder().build(); + subject.handle_bind_message(BindMessage { peer_actors }); + subject.min_hops = Hops::OneHop; + let exit_nodes_before_gossip = subject.init_db_countries(); + + subject.handle_gossip( + standard_gossip, + SocketAddr::from_str("1.1.1.1:1111").unwrap(), + make_cpm_recipient().0, + ); + + assert_eq!(exit_nodes_before_gossip, vec!["US".to_string()]); + assert_eq!( + subject.user_exit_preferences.db_countries, + vec!["FR".to_string(), "US".to_string()] + ); + } + + #[test] + fn introduction_results_in_full_neighborship_in_debutant_db_and_enrich_db_countries_on_one_hop() + { + let debut_node = make_global_cryptde_node_record(1111, true); + let mut debut_subject = neighborhood_from_nodes(&debut_node, None); + debut_subject.min_hops = Hops::OneHop; + let persistent_config = + PersistentConfigurationMock::new().set_past_neighbors_result(Ok(())); + debut_subject.persistent_config_opt = Some(Box::new(persistent_config)); + let debut_root_key = debut_subject.neighborhood_database.root_key().clone(); + let introducer_node = make_node_record(3333, true); //AU + let introducee = make_node_record(2222, true); //FR + let introducer_root_key = introducer_node.public_key().clone(); + let mut introducer_db = debut_subject.neighborhood_database.clone(); + introducer_db.set_root_key(&introducer_root_key); + introducer_db.add_node(introducer_node.clone()).unwrap(); + introducer_db.add_arbitrary_half_neighbor(&introducer_root_key, &debut_root_key); + introducer_db.add_node(introducee.clone()).unwrap(); + introducer_db.add_arbitrary_full_neighbor(&introducer_root_key, introducee.public_key()); + let introduction_gossip = GossipBuilder::new(&introducer_db) + .node(&introducer_root_key, true) + .node(introducee.public_key(), true) + .build(); + let peer_actors = peer_actors_builder().build(); + let exit_nodes_before_gossip = debut_subject.init_db_countries(); + debut_subject.handle_bind_message(BindMessage { peer_actors }); + + debut_subject.handle_gossip( + introduction_gossip, + SocketAddr::from_str("3.3.3.3:3333").unwrap(), + make_cpm_recipient().0, + ); + + assert!(exit_nodes_before_gossip.is_empty()); + assert_eq!( + debut_subject.user_exit_preferences.db_countries, + vec!["AO".to_string()] + ); + } + #[test] #[should_panic( expected = "Neighbor masq://eth-ropsten:AQIDBA@1.2.3.4:1234 is not on the mainnet blockchain" @@ -2952,6 +3610,95 @@ mod tests { assert_eq!(juicy_parts(result_1), (1, 1)); } + #[test] + fn min_hops_change_affects_db_countries_and_exit_location_settings() { + let mut subject = make_standard_subject(); + let root_node_ch = subject.neighborhood_database.root().clone(); + let mut neighbor_one_au = make_node_record(1234, true); + neighbor_one_au.inner.country_code_opt = Some("AU".to_string()); + let mut neighbor_two_fr = make_node_record(2345, true); + neighbor_two_fr.inner.country_code_opt = Some("FR".to_string()); + let mut neighbor_three_cn = make_node_record(3456, true); + neighbor_three_cn.inner.country_code_opt = Some("CN".to_string()); + let mut neighbor_four_us = make_node_record(4567, true); + neighbor_four_us.inner.country_code_opt = Some("US".to_string()); + subject + .neighborhood_database + .add_node(neighbor_one_au.clone()) + .unwrap(); + subject + .neighborhood_database + .add_node(neighbor_two_fr.clone()) + .unwrap(); + subject + .neighborhood_database + .add_node(neighbor_three_cn.clone()) + .unwrap(); + subject + .neighborhood_database + .add_node(neighbor_four_us.clone()) + .unwrap(); + subject + .neighborhood_database + .add_arbitrary_full_neighbor(root_node_ch.public_key(), neighbor_one_au.public_key()); + subject.neighborhood_database.add_arbitrary_full_neighbor( + neighbor_one_au.public_key(), + neighbor_two_fr.public_key(), + ); + subject.neighborhood_database.add_arbitrary_full_neighbor( + neighbor_two_fr.public_key(), + neighbor_three_cn.public_key(), + ); + subject.neighborhood_database.add_arbitrary_full_neighbor( + neighbor_three_cn.public_key(), + neighbor_four_us.public_key(), + ); + subject.user_exit_preferences.db_countries = subject.init_db_countries(); + let exit_locations_by_priority = vec![ExitLocation { + country_codes: vec!["FR".to_string(), "US".to_string()], + priority: 1, + }]; + for exit_location in &exit_locations_by_priority { + subject.enrich_exit_countries(&exit_location.country_codes); + } + subject.user_exit_preferences.fallback_preference = + FallbackPreference::ExitCountryNoFallback; + subject.user_exit_preferences.locations_opt = Some(exit_locations_by_priority); + let tree_hop_db_countries = subject.user_exit_preferences.db_countries.clone(); + let tree_hops_exit_countries = subject.user_exit_preferences.exit_countries.clone(); + let config_msg_two_hops = ConfigChangeMsg { + change: ConfigChange::UpdateMinHops(Hops::TwoHops), + }; + let config_msg_four_hops = ConfigChangeMsg { + change: ConfigChange::UpdateMinHops(Hops::FourHops), + }; + let peer_actors = peer_actors_builder().build(); + subject.handle_bind_message(BindMessage { peer_actors }); + + subject.handle_config_change_msg(config_msg_two_hops); + let two_hops_exit_countries = subject.user_exit_preferences.exit_countries.clone(); + let two_hops_db_countries = subject.user_exit_preferences.db_countries.clone(); + subject.handle_config_change_msg(config_msg_four_hops); + + let four_hops_exit_countries = subject.user_exit_preferences.exit_countries.clone(); + let four_hops_db_countries = subject.user_exit_preferences.db_countries; + assert_eq!( + tree_hop_db_countries, + vec!["CN".to_string(), "US".to_string()] + ); + assert_eq!(tree_hops_exit_countries, vec!["US".to_string()]); + assert_eq!( + two_hops_db_countries, + vec!["CN".to_string(), "FR".to_string(), "US".to_string()] + ); + assert_eq!( + two_hops_exit_countries, + vec!["US".to_string(), "FR".to_string()] + ); + assert_eq!(four_hops_db_countries, vec!["US".to_string()]); + assert_eq!(four_hops_exit_countries, vec!["US".to_string()]); + } + #[test] fn neighborhood_handles_config_change_msg() { assert_handling_of_config_change_msg( @@ -3003,7 +3750,6 @@ mod tests { init_test_logging(); let mut subject = make_standard_subject(); subject.logger = Logger::new("ConfigChange"); - subject.handle_config_change_msg(msg); assertions(&subject); @@ -3031,12 +3777,511 @@ mod tests { subject.set_min_hops_and_patch_size(new_min_hops); - let expected_db_patch_size = Neighborhood::calculate_db_patch_size(new_min_hops); - assert_eq!(subject.min_hops, new_min_hops); - assert_eq!(subject.db_patch_size, expected_db_patch_size); - TestLogHandler::new().exists_log_containing(&format!( - "DEBUG: {test_name}: The value of min_hops (2-hop -> 4-hop) and db_patch_size (3 -> 4) has been changed" - )); + let expected_db_patch_size = Neighborhood::calculate_db_patch_size(new_min_hops); + assert_eq!(subject.min_hops, new_min_hops); + assert_eq!(subject.db_patch_size, expected_db_patch_size); + TestLogHandler::new().exists_log_containing(&format!( + "DEBUG: {test_name}: The value of min_hops (2-hop -> 4-hop) and db_patch_size (3 -> 4) has been changed" + )); + } + + #[test] + fn exit_location_with_multiple_countries_and_priorities_can_be_changed_using_exit_location_msg() + { + init_test_logging(); + let test_name = "exit_location_with_multiple_countries_and_priorities_can_be_changed_using_exit_location_msg"; + let request = UiSetExitLocationRequest { + fallback_routing: true, + exit_locations: vec![ + CountryGroups { + country_codes: vec!["CZ".to_string(), "SK".to_string()], + priority: 1, + }, + CountryGroups { + country_codes: vec!["AT".to_string(), "DE".to_string()], + priority: 2, + }, + CountryGroups { + country_codes: vec!["PL".to_string()], + priority: 3, + }, + ], + show_countries: false, + }; + let message = NodeFromUiMessage { + client_id: 123, + body: request.tmb(234), + }; + let system = System::new(test_name); + let (ui_gateway, _recorder, arc_recorder) = make_recorder(); + let mut subject = make_standard_subject(); + subject.logger = Logger::new(test_name); + let cz = &mut make_node_record(3456, true); + cz.inner.country_code_opt = Some("CZ".to_string()); + let us = &mut make_node_record(4567, true); + us.inner.country_code_opt = Some("US".to_string()); + let sk = &mut make_node_record(5678, true); + sk.inner.country_code_opt = Some("SK".to_string()); + let de = &mut make_node_record(7777, true); + de.inner.country_code_opt = Some("DE".to_string()); + let at = &mut make_node_record(1325, true); + at.inner.country_code_opt = Some("AT".to_string()); + let pl = &mut make_node_record(2543, true); + pl.inner.country_code_opt = Some("PL".to_string()); + let db = &mut subject.neighborhood_database.clone(); + db.add_node(cz.clone()).unwrap(); + db.add_node(de.clone()).unwrap(); + db.add_node(us.clone()).unwrap(); + db.add_node(sk.clone()).unwrap(); + db.add_node(at.clone()).unwrap(); + db.add_node(pl.clone()).unwrap(); + let mut dual_edge = |a: &NodeRecord, b: &NodeRecord| { + db.add_arbitrary_full_neighbor(a.public_key(), b.public_key()); + }; + dual_edge(&subject.neighborhood_database.root(), cz); + dual_edge(cz, de); + dual_edge(cz, us); + dual_edge(us, sk); + dual_edge(us, at); + dual_edge(at, pl); + subject.neighborhood_database = db.clone(); + let subject_addr = subject.start(); + let peer_actors = peer_actors_builder().ui_gateway(ui_gateway).build(); + let cz_public_key = cz.inner.public_key.clone(); + let us_public_key = us.inner.public_key.clone(); + let sk_public_key = sk.inner.public_key.clone(); + let de_public_key = de.inner.public_key.clone(); + let at_public_key = at.inner.public_key.clone(); + let pl_public_key = pl.inner.public_key.clone(); + let assertion_msg = AssertionsMessage { + assertions: Box::new(move |neighborhood: &mut Neighborhood| { + assert_eq!( + neighborhood.user_exit_preferences.exit_countries, + vec!["SK".to_string(), "AT".to_string(), "PL".to_string(),] + ); + assert_eq!( + neighborhood.user_exit_preferences.fallback_preference, + FallbackPreference::ExitCountryWithFallback + ); + assert_eq!( + neighborhood.get_node_country_undesirability(&cz_public_key), + UNREACHABLE_COUNTRY_PENALTY, + "cz We expect {}, country is too close to be exit", + UNREACHABLE_COUNTRY_PENALTY + ); + assert_eq!( + neighborhood.get_node_country_undesirability(&us_public_key), + UNREACHABLE_COUNTRY_PENALTY, + "us We expect {}, country is considered for exit location in fallback", + UNREACHABLE_COUNTRY_PENALTY + ); + assert_eq!( + neighborhood.get_node_country_undesirability(&sk_public_key), + 0u32, + "sk We expect 0, country is with Priority: 1" + ); + assert_eq!( + neighborhood.get_node_country_undesirability(&de_public_key), + UNREACHABLE_COUNTRY_PENALTY, + "de We expect {}, country is too close to be exit", + UNREACHABLE_COUNTRY_PENALTY + ); + assert_eq!( + neighborhood.get_node_country_undesirability(&at_public_key), + 1 * COUNTRY_UNDESIRABILITY_FACTOR, + "at We expect {}, country is with Priority: 2", + 1 * COUNTRY_UNDESIRABILITY_FACTOR + ); + assert_eq!( + neighborhood.get_node_country_undesirability(&pl_public_key), + 2 * COUNTRY_UNDESIRABILITY_FACTOR, + "pl We expect {}, country is with Priority: 3", + 2 * COUNTRY_UNDESIRABILITY_FACTOR + ); + }), + }; + subject_addr.try_send(BindMessage { peer_actors }).unwrap(); + + subject_addr.try_send(message).unwrap(); + subject_addr.try_send(assertion_msg).unwrap(); + + System::current().stop(); + system.run(); + + let recorder_result = arc_recorder.lock().unwrap(); + assert_eq!( + recorder_result.get_record::(0).body, + MessageBody { + opcode: "exitLocation".to_string(), + path: MessagePath::Conversation(234), + payload: Ok("{\"fallbackRouting\":true,\"exitCountrySelection\":[{\"CountryGroups\":[\"CZ\",\"SK\"],\"priority\":1},{\"CountryGroups\":[\"AT\",\"DE\"],\"priority\":2},{\"CountryGroups\":[\"PL\"],\"priority\":3}],\"exitCountries\":null,\"missingCountries\":[\"CZ\",\"DE\"]}".to_string()) + } + ); + assert_eq!( + recorder_result.get_record::(0).target, + MessageTarget::ClientId(123) + ); + TestLogHandler::new().assert_logs_contain_in_order(vec![ + &format!( + "INFO: {}: Fallback Routing is set. Exit location set:", + test_name + ), + &"Country Codes: [\"CZ\", \"SK\"] - Priority: 1; Country Codes: [\"AT\", \"DE\"] - Priority: 2; Country Codes: [\"PL\"] - Priority: 3" + ]); + } + + #[test] + fn no_exit_location_is_set_if_desired_country_codes_not_present_in_neighborhood_with_fallback_routing_set( + ) { + init_test_logging(); + let test_name = "exit_location_with_multiple_countries_and_priorities_can_be_changed_using_exit_location_msg"; + let request = UiSetExitLocationRequest { + fallback_routing: true, + exit_locations: vec![CountryGroups { + country_codes: vec!["CZ".to_string(), "SK".to_string(), "IN".to_string()], + priority: 1, + }], + show_countries: false, + }; + let message = NodeFromUiMessage { + client_id: 234, + body: request.tmb(123), + }; + let system = System::new(test_name); + let (ui_gateway, _recorder, arc_recorder) = make_recorder(); + let mut subject = make_standard_subject(); + subject.min_hops = Hops::TwoHops; + subject.logger = Logger::new(test_name); + let es = &mut make_node_record(3456, true); + es.inner.country_code_opt = Some("ES".to_string()); + let us = &mut make_node_record(4567, true); + us.inner.country_code_opt = Some("US".to_string()); + let hu = &mut make_node_record(5678, true); + hu.inner.country_code_opt = Some("US".to_string()); + let de = &mut make_node_record(7777, true); + de.inner.country_code_opt = Some("DE".to_string()); + let at = &mut make_node_record(1325, true); + at.inner.country_code_opt = Some("AT".to_string()); + let pl = &mut make_node_record(2543, true); + pl.inner.country_code_opt = Some("PL".to_string()); + let db = &mut subject.neighborhood_database.clone(); + db.add_node(es.clone()).unwrap(); + db.add_node(de.clone()).unwrap(); + db.add_node(us.clone()).unwrap(); + db.add_node(hu.clone()).unwrap(); + db.add_node(at.clone()).unwrap(); + db.add_node(pl.clone()).unwrap(); + let mut dual_edge = |a: &NodeRecord, b: &NodeRecord| { + db.add_arbitrary_full_neighbor(a.public_key(), b.public_key()); + }; + dual_edge(&subject.neighborhood_database.root(), es); + dual_edge(es, de); + dual_edge(es, us); + dual_edge(us, hu); + dual_edge(us, at); + dual_edge(at, pl); + subject.neighborhood_database = db.clone(); + let subject_addr = subject.start(); + let peer_actors = peer_actors_builder().ui_gateway(ui_gateway).build(); + let es_public_key = es.inner.public_key.clone(); + let us_public_key = us.inner.public_key.clone(); + let hu_public_key = hu.inner.public_key.clone(); + let de_public_key = de.inner.public_key.clone(); + let at_public_key = at.inner.public_key.clone(); + let pl_public_key = pl.inner.public_key.clone(); + let assertion_msg = AssertionsMessage { + assertions: Box::new(move |neighborhood: &mut Neighborhood| { + assert!(neighborhood.user_exit_preferences.exit_countries.is_empty()); + assert_eq!( + neighborhood.user_exit_preferences.locations_opt, + Some(vec![ExitLocation { + country_codes: vec!["CZ".to_string(), "SK".to_string(), "IN".to_string()], + priority: 1 + }]) + ); + assert_eq!( + neighborhood.user_exit_preferences.db_countries, + vec![ + "AT".to_string(), + "DE".to_string(), + "PL".to_string(), + "US".to_string() + ] + ); + assert_eq!( + neighborhood.user_exit_preferences.fallback_preference, + FallbackPreference::ExitCountryWithFallback + ); + assert_eq!( + neighborhood.get_node_country_undesirability(&es_public_key), + UNREACHABLE_COUNTRY_PENALTY, + "es We expect {}, country is too close to be exit", + UNREACHABLE_COUNTRY_PENALTY + ); + assert_eq!( + neighborhood.get_node_country_undesirability(&us_public_key), + UNREACHABLE_COUNTRY_PENALTY, + "us We expect {}, country is considered for exit location in fallback", + UNREACHABLE_COUNTRY_PENALTY + ); + assert_eq!( + neighborhood.get_node_country_undesirability(&hu_public_key), + UNREACHABLE_COUNTRY_PENALTY, + "hu We expect {}, country is too close to be exit", + UNREACHABLE_COUNTRY_PENALTY + ); + assert_eq!( + neighborhood.get_node_country_undesirability(&de_public_key), + UNREACHABLE_COUNTRY_PENALTY, + "de We expect {}, country is too close to be exit", + UNREACHABLE_COUNTRY_PENALTY + ); + assert_eq!( + neighborhood.get_node_country_undesirability(&at_public_key), + UNREACHABLE_COUNTRY_PENALTY, + "at We expect {}, country is considered for exit location in fallback", + UNREACHABLE_COUNTRY_PENALTY + ); + assert_eq!( + neighborhood.get_node_country_undesirability(&pl_public_key), + UNREACHABLE_COUNTRY_PENALTY, + "pl We expect {}, country is too close to be exit", + UNREACHABLE_COUNTRY_PENALTY + ); + }), + }; + subject_addr.try_send(BindMessage { peer_actors }).unwrap(); + + subject_addr.try_send(message).unwrap(); + + subject_addr.try_send(assertion_msg).unwrap(); + System::current().stop(); + system.run(); + let exit_location_recording = &arc_recorder.lock().unwrap(); + let log_handler = TestLogHandler::new(); + assert_eq!( + exit_location_recording + .get_record::(0) + .body, + MessageBody { + opcode: "exitLocation".to_string(), + path: MessagePath::Conversation(123), + payload: Err((9223372036854775816, "CZ, SK, IN".to_string(),)) + } + ); + assert_eq!( + exit_location_recording + .get_record::(0) + .target, + MessageTarget::ClientId(234) + ); + log_handler.assert_logs_contain_in_order(vec![ + &format!( + "INFO: {}: Fallback Routing is set. Exit location set:", + test_name + ), + &"Country Codes: [\"CZ\", \"SK\", \"IN\"] - Priority: 1", + &format!( + "WARN: {}: Exit Location: following desired countries are missing in Neighborhood [\"CZ\", \"SK\", \"IN\"]", + test_name + ), + ]); + } + + #[test] + fn exit_location_is_set_and_unset_with_fallback_routing_using_exit_location_msg() { + init_test_logging(); + let test_name = + "exit_location_is_set_and_unset_with_fallback_routing_using_exit_location_msg"; + let request = UiSetExitLocationRequest { + fallback_routing: false, + exit_locations: vec![ + CountryGroups { + country_codes: vec!["CZ".to_string()], + priority: 1, + }, + CountryGroups { + country_codes: vec!["FR".to_string()], + priority: 2, + }, + ], + show_countries: false, + }; + let set_exit_location_message = NodeFromUiMessage { + client_id: 8765, + body: request.tmb(1234), + }; + let request_2 = UiSetExitLocationRequest { + fallback_routing: true, + exit_locations: vec![], + show_countries: false, + }; + let clear_exit_location_message = NodeFromUiMessage { + client_id: 6543, + body: request_2.tmb(7894), + }; + let mut subject = make_standard_subject(); + let system = System::new(test_name); + let (ui_gateway, _, ui_gateway_recording_arc) = make_recorder(); + subject.logger = Logger::new(test_name); + let cz = &mut make_node_record(3456, true); + cz.inner.country_code_opt = Some("CZ".to_string()); + let standard_node_1 = &make_node_record(4567, true); + let fr = &mut make_node_record(5678, true); + fr.inner.country_code_opt = Some("FR".to_string()); + let standard_node_2 = &make_node_record(7777, true); + let root_node = subject.neighborhood_database.root().clone(); + let db = &mut subject.neighborhood_database; + db.add_node(cz.clone()).unwrap(); + db.add_node(standard_node_2.clone()).unwrap(); + db.add_node(standard_node_1.clone()).unwrap(); + db.add_node(fr.clone()).unwrap(); + let mut dual_edge = |a: &NodeRecord, b: &NodeRecord| { + db.add_arbitrary_full_neighbor(a.public_key(), b.public_key()); + }; + dual_edge(&root_node, cz); + dual_edge(cz, standard_node_2); + dual_edge(cz, standard_node_1); + dual_edge(standard_node_1, fr); + subject.neighborhood_database = db.clone(); + let subject_addr = subject.start(); + let peer_actors = peer_actors_builder().ui_gateway(ui_gateway).build(); + let cz_public_key = cz.inner.public_key.clone(); + let sn_1_public_key = standard_node_1.inner.public_key.clone(); + let fr_public_key = fr.inner.public_key.clone(); + let sn_2_public_key = standard_node_2.inner.public_key.clone(); + let assert_country_undesirability_populated = AssertionsMessage { + assertions: Box::new(move |neighborhood: &mut Neighborhood| { + assert_eq!( + neighborhood.get_node_country_undesirability(&cz_public_key), + 0u32, + "CZ - We expect zero, country is with Priority: 1" + ); + assert_eq!( + neighborhood.get_node_country_undesirability(&sn_1_public_key), + 0u32, + "We expect 0, country is not considered for exit location, so country_undesirability doesn't matter" + ); + assert_eq!( + neighborhood.get_node_country_undesirability(&fr_public_key), + 1 * COUNTRY_UNDESIRABILITY_FACTOR, + "FR - We expect {}, country is with Priority: 2", + 1 * COUNTRY_UNDESIRABILITY_FACTOR + ); + assert_eq!( + neighborhood.get_node_country_undesirability(&sn_2_public_key), + UNREACHABLE_COUNTRY_PENALTY, + "We expect 100M, country is not considered for exit location, so is unreachable" + ); + assert_eq!( + neighborhood.user_exit_preferences.exit_countries, + vec!["FR".to_string()] + ); + assert_eq!( + neighborhood.user_exit_preferences.fallback_preference, + FallbackPreference::ExitCountryNoFallback + ); + }), + }; + let cz_public_key_2 = cz.inner.public_key.clone(); + let r_public_key_2 = standard_node_1.inner.public_key.clone(); + let fr_public_key_2 = fr.inner.public_key.clone(); + let t_public_key_2 = standard_node_2.inner.public_key.clone(); + let assert_country_undesirability_and_exit_preference_cleared = AssertionsMessage { + assertions: Box::new(move |neighborhood: &mut Neighborhood| { + assert_eq!( + neighborhood.get_node_country_undesirability(&cz_public_key_2), + 0u32, + "We expect zero, exit_location was unset" + ); + assert_eq!( + neighborhood.get_node_country_undesirability(&r_public_key_2), + 0u32, + "We expect zero, exit_location was unset" + ); + assert_eq!( + neighborhood.get_node_country_undesirability(&fr_public_key_2), + 0u32, + "We expect zero, exit_location was unset" + ); + assert_eq!( + neighborhood.get_node_country_undesirability(&t_public_key_2), + 0u32, + "We expect zero, exit_location was unset" + ); + assert_eq!( + neighborhood.user_exit_preferences.exit_countries.is_empty(), + true + ); + assert_eq!( + neighborhood.user_exit_preferences.fallback_preference, + FallbackPreference::Nothing + ) + }), + }; + + subject_addr.try_send(BindMessage { peer_actors }).unwrap(); + subject_addr.try_send(set_exit_location_message).unwrap(); + subject_addr + .try_send(assert_country_undesirability_populated) + .unwrap(); + subject_addr.try_send(clear_exit_location_message).unwrap(); + subject_addr + .try_send(assert_country_undesirability_and_exit_preference_cleared) + .unwrap(); + + System::current().stop(); + system.run(); + let ui_gateway_recording = ui_gateway_recording_arc.lock().unwrap(); + let record_one: &NodeToUiMessage = ui_gateway_recording.get_record(0); + let record_two: &NodeToUiMessage = ui_gateway_recording.get_record(1); + assert_eq!(ui_gateway_recording.len(), 2); + assert_eq!( + record_one.body, + UiSetExitLocationResponse { + fallback_routing: false, + exit_country_selection: vec![ + ExitLocation { + country_codes: vec!["CZ".to_string()], + priority: 1 + }, + ExitLocation { + country_codes: vec!["FR".to_string()], + priority: 2 + } + ], + exit_countries: None, + missing_countries: vec!["CZ".to_string()], + } + .tmb(1234) + ); + assert_eq!( + record_two, + &NodeToUiMessage { + target: MessageTarget::ClientId(6543), + body: UiSetExitLocationResponse { + fallback_routing: true, + exit_country_selection: vec![], + exit_countries: None, + missing_countries: vec![], + } + .tmb(7894), + } + ); + TestLogHandler::new().assert_logs_contain_in_order(vec![ + &format!( + "INFO: {}: Fallback Routing NOT set. Exit location set: Country Codes: [\"CZ\"] - Priority: 1; Country Codes: [\"FR\"] - Priority: 2", + test_name + ), + &format!( + "WARN: {}: Exit Location: following desired countries are missing in Neighborhood [\"CZ\"]", + test_name + ), + &format!( + "INFO: {}: Fallback Routing is set. Exit location unset.", + test_name + ), + ]); } #[test] @@ -3196,6 +4441,79 @@ mod tests { ); } + /* + Database: + + A---B---C---D---E + | | | | | + F---G---H---I---J + | | | | | + K---L---M---N---O + | | | | | + P---Q---R---S---T + | | | | | + U---V---W---X---Y + + All these Nodes are standard-mode. L is the root Node. + */ + #[test] + fn find_exit_locations_in_packed_grid() { + let mut subject = make_standard_subject(); + let db = &mut subject.neighborhood_database; + let keys = make_db_with_regular_5_x_5_network(db); + designate_root_node(db, keys.get("l").unwrap()); + + let mut exit_nodes = subject.find_exit_locations(keys.get("l").unwrap(), 3); + + let total_exit_nodes = exit_nodes.len(); + exit_nodes.sort(); + exit_nodes.dedup(); + let dedup_len = exit_nodes.len(); + assert_eq!(total_exit_nodes, dedup_len); + assert_eq!(total_exit_nodes, 20); + } + + #[test] + fn find_exit_locations_in_row_structure() { + let mut subject = make_standard_subject(); + let db = &mut subject.neighborhood_database; + let mut generator = 1000; + let mut make_node = |db: &mut NeighborhoodDatabase| { + let node = &db.add_node(make_node_record(generator, true)).unwrap(); + generator += 1; + node.clone() + }; + let n1 = make_node(db); + let n2 = make_node(db); + let n3 = make_node(db); + let n4 = make_node(db); + let n5 = make_node(db); + let f1 = make_node(db); + let f2 = make_node(db); + let f3 = make_node(db); + let f4 = make_node(db); + let f5 = make_node(db); + db.add_arbitrary_full_neighbor(&n1, &n2); + db.add_arbitrary_full_neighbor(&n2, &n3); + db.add_arbitrary_full_neighbor(&n3, &n4); + db.add_arbitrary_full_neighbor(&n4, &n5); + db.add_arbitrary_full_neighbor(&n5, &f1); + db.add_arbitrary_full_neighbor(&f1, &f2); + db.add_arbitrary_full_neighbor(&f2, &f3); + db.add_arbitrary_full_neighbor(&f3, &f4); + db.add_arbitrary_full_neighbor(&f4, &f5); + designate_root_node(db, &n1); + + let mut exit_nodes = subject.find_exit_locations(&n1, 3); + + let total_exit_nodes = exit_nodes.len(); + exit_nodes.sort(); + exit_nodes.dedup(); + let dedup_len = exit_nodes.len(); + assert_eq!(total_exit_nodes, dedup_len); + assert_eq!(total_exit_nodes, 7); + } + /* Database: @@ -3276,62 +4594,113 @@ mod tests { */ #[test] - fn route_optimization_test() { + fn route_optimization_by_serving_rates() { let mut subject = make_standard_subject(); let db = &mut subject.neighborhood_database; - let mut generator = 1000; - let mut make_node = |db: &mut NeighborhoodDatabase| { - let node = &db.add_node(make_node_record(generator, true)).unwrap(); - generator += 1; - node.clone() - }; - let mut make_row = |db: &mut NeighborhoodDatabase| { - let n1 = make_node(db); - let n2 = make_node(db); - let n3 = make_node(db); - let n4 = make_node(db); - let n5 = make_node(db); - db.add_arbitrary_full_neighbor(&n1, &n2); - db.add_arbitrary_full_neighbor(&n2, &n3); - db.add_arbitrary_full_neighbor(&n3, &n4); - db.add_arbitrary_full_neighbor(&n4, &n5); - (n1, n2, n3, n4, n5) - }; - let join_rows = |db: &mut NeighborhoodDatabase, first_row, second_row| { - let (f1, f2, f3, f4, f5) = first_row; - let (s1, s2, s3, s4, s5) = second_row; - db.add_arbitrary_full_neighbor(f1, s1); - db.add_arbitrary_full_neighbor(f2, s2); - db.add_arbitrary_full_neighbor(f3, s3); - db.add_arbitrary_full_neighbor(f4, s4); - db.add_arbitrary_full_neighbor(f5, s5); + let (recipient, _) = make_node_to_ui_recipient(); + subject.node_to_ui_recipient_opt = Some(recipient); + let message = UiSetExitLocationRequest { + fallback_routing: true, + exit_locations: vec![], + show_countries: false, }; - let designate_root_node = |db: &mut NeighborhoodDatabase, key| { - let root_node_key = db.root().public_key().clone(); - let node = db.node_by_key(key).unwrap().clone(); - db.root_mut().inner = node.inner.clone(); - db.root_mut().metadata = node.metadata.clone(); - db.remove_node(&root_node_key); - }; - let (a, b, c, d, e) = make_row(db); - let (f, g, h, i, j) = make_row(db); - let (k, l, m, n, o) = make_row(db); - let (p, q, r, s, t) = make_row(db); - let (u, v, w, x, y) = make_row(db); - join_rows(db, (&a, &b, &c, &d, &e), (&f, &g, &h, &i, &j)); - join_rows(db, (&f, &g, &h, &i, &j), (&k, &l, &m, &n, &o)); - join_rows(db, (&k, &l, &m, &n, &o), (&p, &q, &r, &s, &t)); - join_rows(db, (&p, &q, &r, &s, &t), (&u, &v, &w, &x, &y)); - designate_root_node(db, &l); + let keys = make_db_with_regular_5_x_5_network(db); + designate_root_node(db, keys.get("l").unwrap()); + subject.handle_exit_location_message(message, 0, 0); let before = Instant::now(); // All the target-designated routes from L to N let route = subject - .find_best_route_segment(&l, Some(&n), 3, 10000, RouteDirection::Back, None) + .find_best_route_segment( + &keys.get("l").unwrap(), + Some(&keys.get("n").unwrap()), + 3, + 10000, + RouteDirection::Back, + None, + ) .unwrap(); let after = Instant::now(); - assert_eq!(route, vec![&l, &g, &h, &i, &n]); // Cheaper than [&l, &q, &r, &s, &n] + assert_eq!( + route, + vec![ + keys.get("l").unwrap(), + keys.get("g").unwrap(), + keys.get("h").unwrap(), + keys.get("i").unwrap(), + keys.get("n").unwrap() + ] + ); // Cheaper than [&l, &q, &r, &s, &n] + let interval = after.duration_since(before); + assert!( + interval.as_millis() <= 100, + "Should have calculated route in <=100ms, but was {}ms", + interval.as_millis() + ); + } + + /* Complex testing of country_undesirability on large network with aim to find fallback routing and non fallback routing mechanisms + + Database: + + A---B---C---D---E + | | | | | + F---G---H---I---J + | | | | | + K---L---M---N---O + | | | | | + P---Q---R---S---T + | | | | | + U---V---W---X---Y + + All these Nodes are standard-mode. L is the root Node. C and T are "CZ" standard nodes + + */ + #[test] + fn route_optimization_with_user_exit_preferences() { + let mut subject = make_standard_subject(); + subject.min_hops = Hops::TwoHops; + let db = &mut subject.neighborhood_database; + let (recipient, _) = make_node_to_ui_recipient(); + subject.node_to_ui_recipient_opt = Some(recipient); + let message = UiSetExitLocationRequest { + fallback_routing: false, + exit_locations: vec![CountryGroups { + country_codes: vec!["CZ".to_string()], + priority: 1, + }], + show_countries: false, + }; + let keys = make_db_with_regular_5_x_5_network(db); + db.node_by_key_mut(&keys.get("c").unwrap()) + .unwrap() + .inner + .country_code_opt = Some("CZ".to_string()); + db.node_by_key_mut(&keys.get("t").unwrap()) + .unwrap() + .inner + .country_code_opt = Some("CZ".to_string()); + let control_db = db.clone(); + designate_root_node(db, &keys.get("l").unwrap()); + subject.handle_exit_location_message(message, 0, 0); + let before = Instant::now(); + + let route_cz = subject.find_best_route_segment( + &keys.get("l").unwrap(), + None, + 3, + 10000, + RouteDirection::Over, + None, + ); + + let after = Instant::now(); + let exit_node = control_db.node_by_key(&route_cz.as_ref().unwrap().last().unwrap()); + assert_eq!( + exit_node.unwrap().inner.country_code_opt, + Some("CZ".to_string()) + ); let interval = after.duration_since(before); assert!( interval.as_millis() <= 100, @@ -3343,11 +4712,157 @@ mod tests { /* Database: - P---q---R - - Test is written from the standpoint of P. Node q is non-routing. + root_key---c_au---b_fr + | + a_fr + Test is written from the standpoint of root_key. */ + #[test] + fn exit_node_not_found_due_to_country_code_strict_requirement() { + let mut subject = make_standard_subject(); + let (recipient, _) = make_node_to_ui_recipient(); + subject.node_to_ui_recipient_opt = Some(recipient); + subject.user_exit_preferences.fallback_preference = + FallbackPreference::ExitCountryWithFallback; + let message = UiSetExitLocationRequest { + fallback_routing: false, + exit_locations: vec![CountryGroups { + country_codes: vec!["CZ".to_string()], + priority: 1, + }], + show_countries: false, + }; + let db = &mut subject.neighborhood_database; + let root_key = &db.root_mut().public_key().clone(); + let a_fr = &db.add_node(make_node_record(2345, true)).unwrap(); + let b_fr = &db.add_node(make_node_record(5678, true)).unwrap(); + let c_au = &db.add_node(make_node_record(1234, true)).unwrap(); + db.add_arbitrary_full_neighbor(root_key, c_au); + db.add_arbitrary_full_neighbor(c_au, b_fr); + db.add_arbitrary_full_neighbor(c_au, a_fr); + subject.handle_exit_location_message(message, 0, 0); + + let route_cz = + subject.find_best_route_segment(root_key, None, 2, 10000, RouteDirection::Over, None); + + assert_eq!(route_cz, None); + } + + #[test] + fn route_for_au_country_code_is_constructed_with_fallback_routing() { + let mut subject = make_standard_subject(); + let root_key = &subject + .neighborhood_database + .root_mut() + .public_key() + .clone(); + let mut a_fr_node = make_node_record(2345, true); + a_fr_node.inner.rate_pack.exit_byte_rate = 1; + a_fr_node.inner.rate_pack.exit_service_rate = 1; + let mut c_au_node = make_node_record(1234, true); + c_au_node.inner.rate_pack.exit_byte_rate = 10; + c_au_node.inner.rate_pack.exit_service_rate = 10; + let a_fr_key = &subject.neighborhood_database.add_node(a_fr_node).unwrap(); + let b_fr_key = &subject + .neighborhood_database + .add_node(make_node_record(5678, true)) + .unwrap(); + let c_au_key = &subject.neighborhood_database.add_node(c_au_node).unwrap(); + subject + .neighborhood_database + .add_arbitrary_full_neighbor(root_key, b_fr_key); + subject + .neighborhood_database + .add_arbitrary_full_neighbor(b_fr_key, c_au_key); + subject + .neighborhood_database + .add_arbitrary_full_neighbor(b_fr_key, a_fr_key); + subject + .neighborhood_database + .add_arbitrary_full_neighbor(a_fr_key, c_au_key); + let cdb = subject.neighborhood_database.clone(); + let (recipient, _) = make_node_to_ui_recipient(); + subject.node_to_ui_recipient_opt = Some(recipient); + let message = UiSetExitLocationRequest { + fallback_routing: true, + exit_locations: vec![CountryGroups { + country_codes: vec!["AO".to_string()], + priority: 1, + }], + show_countries: false, + }; + subject.handle_exit_location_message(message, 0, 0); + let subject_min_hops = 2; + + let route_au = subject.find_best_route_segment( + root_key, + None, + subject_min_hops, + 10000, + RouteDirection::Over, + None, + ); + + let exit_node = cdb.node_by_key(&route_au.as_ref().unwrap().last().unwrap()); + assert_eq!( + exit_node.unwrap().inner.country_code_opt, + Some("AO".to_string()) + ); + } + + #[test] + fn route_for_fr_country_code_is_constructed_without_fallback_routing() { + let mut subject = make_standard_subject(); + let root_key = &subject + .neighborhood_database + .root_mut() + .public_key() + .clone(); + let mut a_fr_node = make_node_record(2345, true); + a_fr_node.inner.country_code_opt = Some("FR".to_string()); + let a_fr = &subject.neighborhood_database.add_node(a_fr_node).unwrap(); + let mut b_fr_node = make_node_record(5678, true); + b_fr_node.inner.country_code_opt = Some("FR".to_string()); + let b_fr = &subject.neighborhood_database.add_node(b_fr_node).unwrap(); + let mut c_au_node = make_node_record(1234, true); + c_au_node.inner.country_code_opt = Some("AU".to_string()); + let c_au = &subject.neighborhood_database.add_node(c_au_node).unwrap(); + subject + .neighborhood_database + .add_arbitrary_full_neighbor(root_key, b_fr); + subject + .neighborhood_database + .add_arbitrary_full_neighbor(b_fr, c_au); + subject + .neighborhood_database + .add_arbitrary_full_neighbor(b_fr, a_fr); + subject + .neighborhood_database + .add_arbitrary_full_neighbor(a_fr, c_au); + let cdb = subject.neighborhood_database.clone(); + let (recipient, _) = make_node_to_ui_recipient(); + subject.node_to_ui_recipient_opt = Some(recipient); + let message = UiSetExitLocationRequest { + fallback_routing: false, + exit_locations: vec![CountryGroups { + country_codes: vec!["FR".to_string()], + priority: 1, + }], + show_countries: false, + }; + subject.handle_exit_location_message(message, 0, 0); + + let route_fr = + subject.find_best_route_segment(root_key, None, 2, 10000, RouteDirection::Over, None); + + let exit_node = cdb.node_by_key(&route_fr.as_ref().unwrap().last().unwrap()); + assert_eq!( + exit_node.unwrap().inner.country_code_opt, + Some("FR".to_string()) + ); + } + #[test] fn cant_route_through_non_routing_node() { let mut subject = make_standard_subject(); @@ -3444,7 +4959,7 @@ mod tests { TestLogHandler::new().exists_log_containing( "TRACE: Neighborhood: Node with PubKey 0x02030405 \ failed to reach host \"hostname.com\" during ExitRequest; \ - Undesirability: 2350745 + 100000000 = 102350745", + Undesirability: 2350745 + 100000000 + 0 = 102350745", ); } @@ -3530,6 +5045,7 @@ mod tests { 100, true, true, + None, ); let this_node_inside = this_node.clone(); let removed_neighbor = make_node_record(2345, true); @@ -3657,7 +5173,7 @@ mod tests { let gossip_acceptor = GossipAcceptorMock::new() .handle_params(&handle_params_arc) .handle_result(GossipAcceptanceResult::Ignored); - let subject_node = make_global_cryptde_node_record(1234, true); // 9e7p7un06eHs6frl5A + let mut subject_node = make_global_cryptde_node_record(1234, true); // 9e7p7un06eHs6frl5A let neighbor = make_node_record(1111, true); let mut subject = neighborhood_from_nodes(&subject_node, Some(&neighbor)); subject.gossip_acceptor = Box::new(gossip_acceptor); @@ -3683,6 +5199,7 @@ mod tests { let (call_database, call_agrs, call_gossip_source, neighborhood_metadata) = handle_params.remove(0); assert!(handle_params.is_empty()); + subject_node.metadata.last_update = call_database.root().metadata.last_update; assert_eq!(&subject_node, call_database.root()); assert_eq!(1, call_database.keys().len()); let agrs: Vec = gossip.try_into().unwrap(); @@ -4306,17 +5823,40 @@ mod tests { } #[test] - fn handle_new_public_ip_changes_public_ip_and_nothing_else() { + fn handle_new_public_ip_changes_public_ip_and_country_code_nothing_else() { init_test_logging(); let subject_node = make_global_cryptde_node_record(1234, true); let neighbor = make_node_record(1050, true); let mut subject: Neighborhood = neighborhood_from_nodes(&subject_node, Some(&neighbor)); + subject + .neighborhood_database + .root_mut() + .inner + .country_code_opt = Some("AU".to_string()); let new_public_ip = IpAddr::from_str("4.3.2.1").unwrap(); subject.handle_new_public_ip(NewPublicIp { new_ip: new_public_ip, }); + assert_eq!( + subject.neighborhood_database.root().inner.country_code_opt, + Some("AO".to_string()) + ); + assert_eq!( + subject.neighborhood_database.root().inner.country_code_opt, + Some( + subject + .neighborhood_database + .root() + .metadata + .node_location_opt + .as_ref() + .unwrap() + .country_code + .clone() + ) + ); assert_eq!( subject .neighborhood_database @@ -4637,6 +6177,7 @@ mod tests { 100, true, true, + None, ); let mut db = db_from_node(&this_node); let far_neighbor = make_node_record(1324, true); @@ -4696,12 +6237,12 @@ mod tests { }); let tlh = TestLogHandler::new(); tlh.await_log_containing( - &format!("\"BAYFBw\" [label=\"AR v0\\nBAYFBw\\n4.6.5.7:4657\"];"), + &format!("\"BAYFBw\" [label=\"AR v0 AO\\nBAYFBw\\n4.6.5.7:4657\"];"), 5000, ); tlh.exists_log_containing("Received Gossip: digraph db { "); - tlh.exists_log_containing("\"AQMCBA\" [label=\"AR v0\\nAQMCBA\"];"); + tlh.exists_log_containing("\"AQMCBA\" [label=\"AR v0 AO\\nAQMCBA\"];"); tlh.exists_log_containing(&format!( "\"{}\" [label=\"{}\"] [shape=none];", cryptde.public_key(), @@ -4774,6 +6315,7 @@ mod tests { }; let temp_db = db_from_node(&this_node); let expected_gnr = GossipNodeRecord::from((&temp_db, this_node.public_key(), true)); + assert_contains(&gossip.node_records, &expected_gnr); assert_eq!(1, gossip.node_records.len()); TestLogHandler::new().exists_log_containing(&format!( @@ -5521,15 +7063,15 @@ mod tests { init_test_logging(); let subject_node = make_global_cryptde_node_record(1345, true); let public_key = PublicKey::from(&b"exit_node"[..]); - let node_record = NodeRecord::new( - &public_key, - make_wallet("earning"), - rate_pack(100), - true, - true, - 0, - main_cryptde(), - ); + let node_record_inputs = NodeRecordInputs { + earning_wallet: make_wallet("earning"), + rate_pack: rate_pack(100), + accepts_connections: true, + routes_data: true, + version: 0, + location_opt: None, + }; + let node_record = NodeRecord::new(&public_key, main_cryptde(), node_record_inputs); let unreachable_host = String::from("facebook.com"); let mut subject = neighborhood_from_nodes(&subject_node, None); let _ = subject.neighborhood_database.add_node(node_record); @@ -6124,4 +7666,93 @@ mod tests { neighborhood } + + /* + Database: + + + A---B---C---D---E + | | | | | + F---G---H---I---J + | | | | | + K---L---M---N---O + | | | | | + P---Q---R---S---T + | | | | | + U---V---W---X---Y + */ + fn make_db_with_regular_5_x_5_network( + db: &mut NeighborhoodDatabase, + ) -> HashMap<&'static str, PublicKey> { + let mut generator = 1000; + let mut make_node = |db: &mut NeighborhoodDatabase| { + let node = &db.add_node(make_node_record(generator, true)).unwrap(); + generator += 1; + node.clone() + }; + let mut make_row = |db: &mut NeighborhoodDatabase| { + let n1 = make_node(db); + let n2 = make_node(db); + let n3 = make_node(db); + let n4 = make_node(db); + let n5 = make_node(db); + db.add_arbitrary_full_neighbor(&n1, &n2); + db.add_arbitrary_full_neighbor(&n2, &n3); + db.add_arbitrary_full_neighbor(&n3, &n4); + db.add_arbitrary_full_neighbor(&n4, &n5); + (n1, n2, n3, n4, n5) + }; + let join_rows = |db: &mut NeighborhoodDatabase, first_row, second_row| { + let (f1, f2, f3, f4, f5) = first_row; + let (s1, s2, s3, s4, s5) = second_row; + db.add_arbitrary_full_neighbor(f1, s1); + db.add_arbitrary_full_neighbor(f2, s2); + db.add_arbitrary_full_neighbor(f3, s3); + db.add_arbitrary_full_neighbor(f4, s4); + db.add_arbitrary_full_neighbor(f5, s5); + }; + let (a, b, c, d, e) = make_row(db); + let (f, g, h, i, j) = make_row(db); + let (k, l, m, n, o) = make_row(db); + let (p, q, r, s, t) = make_row(db); + let (u, v, w, x, y) = make_row(db); + join_rows(db, (&a, &b, &c, &d, &e), (&f, &g, &h, &i, &j)); + join_rows(db, (&f, &g, &h, &i, &j), (&k, &l, &m, &n, &o)); + join_rows(db, (&k, &l, &m, &n, &o), (&p, &q, &r, &s, &t)); + join_rows(db, (&p, &q, &r, &s, &t), (&u, &v, &w, &x, &y)); + let keypairs = [ + ("a", a), + ("b", b), + ("c", c), + ("d", d), + ("e", e), + ("f", f), + ("g", g), + ("h", h), + ("i", i), + ("j", j), + ("k", k), + ("l", l), + ("m", m), + ("n", n), + ("o", o), + ("p", p), + ("q", q), + ("r", r), + ("s", s), + ("t", t), + ("u", u), + ("v", v), + ("w", w), + ("x", x), + ("y", y), + ]; + HashMap::from_iter(keypairs) + } + + fn designate_root_node(db: &mut NeighborhoodDatabase, key: &PublicKey) { + let root_node_key = db.root_key().clone(); + db.set_root_key(key); + db.remove_node(&root_node_key); + } } diff --git a/node/src/neighborhood/neighborhood_database.rs b/node/src/neighborhood/neighborhood_database.rs index ceabe133a..7255015b4 100644 --- a/node/src/neighborhood/neighborhood_database.rs +++ b/node/src/neighborhood/neighborhood_database.rs @@ -3,7 +3,8 @@ use super::neighborhood_database::NeighborhoodDatabaseError::NodeKeyNotFound; use crate::neighborhood::dot_graph::{ render_dot_graph, DotRenderable, EdgeRenderable, NodeRenderable, NodeRenderableInner, }; -use crate::neighborhood::node_record::{NodeRecord, NodeRecordError}; +use crate::neighborhood::node_location::get_node_location; +use crate::neighborhood::node_record::{NodeRecord, NodeRecordError, NodeRecordInputs}; use crate::sub_lib::cryptde::CryptDE; use crate::sub_lib::cryptde::PublicKey; use crate::sub_lib::neighborhood::NeighborhoodMode; @@ -24,6 +25,9 @@ pub const ISOLATED_NODE_GRACE_PERIOD_SECS: u32 = 30; #[derive(Clone)] pub struct NeighborhoodDatabase { + #[cfg(test)] + pub this_node: PublicKey, // Public only in tests + #[cfg(not(test))] this_node: PublicKey, by_public_key: HashMap, by_ip_addr: HashMap, @@ -49,16 +53,19 @@ impl NeighborhoodDatabase { by_ip_addr: HashMap::new(), logger: Logger::new("NeighborhoodDatabase"), }; - - let mut node_record = NodeRecord::new( - public_key, + let location_opt = match neighborhood_mode.node_addr_opt() { + Some(node_addr) => get_node_location(Some(node_addr.ip_addr())), + None => None, + }; + let node_record_data = NodeRecordInputs { earning_wallet, - *neighborhood_mode.rate_pack(), - neighborhood_mode.accepts_connections(), - neighborhood_mode.routes_data(), - 0, - cryptde, - ); + rate_pack: *neighborhood_mode.rate_pack(), + accepts_connections: neighborhood_mode.accepts_connections(), + routes_data: neighborhood_mode.routes_data(), + version: 0, + location_opt, + }; + let mut node_record = NodeRecord::new(public_key, cryptde, node_record_data); if let Some(node_addr) = neighborhood_mode.node_addr_opt() { node_record .set_node_addr(&node_addr) @@ -73,6 +80,10 @@ impl NeighborhoodDatabase { self.node_by_key(&self.this_node).expect("Internal error") } + pub fn root_key(&self) -> &PublicKey { + &self.this_node + } + pub fn root_mut(&mut self) -> &mut NodeRecord { let root_key = &self.this_node.clone(); self.node_by_key_mut(root_key).expect("Internal error") @@ -90,6 +101,13 @@ impl NeighborhoodDatabase { self.by_public_key.get_mut(public_key) } + pub fn nodes_mut(&mut self) -> Vec<&mut NodeRecord> { + self.by_public_key + .iter_mut() + .map(|(_key, node_record)| node_record) + .collect() + } + pub fn node_by_ip(&self, ip_addr: &IpAddr) -> Option<&NodeRecord> { match self.by_ip_addr.get(ip_addr) { Some(key) => self.node_by_key(key), @@ -284,11 +302,16 @@ impl NeighborhoodDatabase { to: k.clone(), }) }); + let country_code = match &nr.inner.country_code_opt { + Some(cc) => cc.clone(), + None => "ZZ".to_string(), + }; node_renderables.push(NodeRenderable { inner: Some(NodeRenderableInner { version: nr.version(), accepts_connections: nr.accepts_connections(), routes_data: nr.routes_data(), + country_code, }), public_key: public_key.clone(), node_addr: nr.node_addr_opt(), @@ -362,10 +385,13 @@ pub enum NeighborhoodDatabaseError { #[cfg(test)] mod tests { use super::*; + use crate::neighborhood::node_location::NodeLocation; use crate::sub_lib::cryptde_null::CryptDENull; use crate::sub_lib::utils::time_t_timestamp; use crate::test_utils::assert_string_contains; - use crate::test_utils::neighborhood_test_utils::{db_from_node, make_node_record}; + use crate::test_utils::neighborhood_test_utils::{ + db_from_node, make_node_record, make_segmented_ip, make_segments, + }; use masq_lib::constants::DEFAULT_CHAIN; use masq_lib::test_utils::utils::TEST_DEFAULT_CHAIN; use std::iter::FromIterator; @@ -378,10 +404,12 @@ mod tests { #[test] fn a_brand_new_database_has_the_expected_contents() { - let this_node = make_node_record(1234, true); + let mut this_node = make_node_record(1234, true); let subject = db_from_node(&this_node); + let last_update = subject.root().metadata.last_update; + this_node.metadata.last_update = last_update; assert_eq!(subject.this_node, this_node.public_key().clone()); assert_eq!( subject.by_public_key, @@ -406,7 +434,7 @@ mod tests { #[test] fn can_get_mutable_root() { - let this_node = make_node_record(1234, true); + let mut this_node = make_node_record(1234, true); let mut subject = NeighborhoodDatabase::new( this_node.public_key(), @@ -415,6 +443,8 @@ mod tests { &CryptDENull::from(this_node.public_key(), TEST_DEFAULT_CHAIN), ); + let last_update = subject.root().metadata.last_update; + this_node.metadata.last_update = last_update; assert_eq!(subject.this_node, this_node.public_key().clone()); assert_eq!( subject.by_public_key, @@ -473,7 +503,7 @@ mod tests { #[test] fn node_by_key_works() { - let this_node = make_node_record(1234, true); + let mut this_node = make_node_record(1234, true); let one_node = make_node_record(4567, true); let another_node = make_node_record(5678, true); let mut subject = NeighborhoodDatabase::new( @@ -485,6 +515,9 @@ mod tests { subject.add_node(one_node.clone()).unwrap(); + let this_pubkey = this_node.public_key(); + let updated_record = subject.node_by_key(this_pubkey).unwrap(); + this_node.metadata.last_update = updated_record.metadata.last_update; assert_eq!( subject.node_by_key(this_node.public_key()).unwrap().clone(), this_node @@ -498,13 +531,21 @@ mod tests { #[test] fn node_by_ip_works() { - let this_node = make_node_record(1234, true); + let mut this_node = make_node_record(1234, true); + this_node.inner.country_code_opt = Some("AD".to_string()); + this_node.metadata.node_location_opt = Some(NodeLocation { + country_code: "AD".to_string(), + }); + this_node.resign(); let one_node = make_node_record(4567, true); let another_node = make_node_record(5678, true); let mut subject = db_from_node(&this_node); subject.add_node(one_node.clone()).unwrap(); + let this_pubkey = this_node.public_key(); + let updated_record = subject.node_by_key(this_pubkey).unwrap(); + this_node.metadata.last_update = updated_record.metadata.last_update; assert_eq!( subject .node_by_ip(&this_node.node_addr_opt().unwrap().ip_addr()) @@ -525,6 +566,49 @@ mod tests { ); } + #[test] + fn nodes_mut_works() { + let root_node = make_node_record(1234, true); + let node_a = make_node_record(2345, false); + let node_b = make_node_record(3456, true); + let mut subject = NeighborhoodDatabase::new( + root_node.public_key(), + (&root_node).into(), + Wallet::from_str("0x0000000000000000000000000000000000004444").unwrap(), + &CryptDENull::from(root_node.public_key(), TEST_DEFAULT_CHAIN), + ); + subject.add_node(node_a.clone()).unwrap(); + subject.add_node(node_b.clone()).unwrap(); + let mut ipnumber: u16 = 7890; + let mut keys_nums: Vec<(PublicKey, u16)> = vec![]; + + let mutable_nodes = subject.nodes_mut(); + for node in mutable_nodes { + let (seg1, seg2, seg3, seg4) = make_segments(ipnumber); + node.metadata.node_addr_opt = Some(NodeAddr::new( + &make_segmented_ip(seg1, seg2, seg3, seg4), + &[ipnumber], + )); + keys_nums.push((node.inner.public_key.clone(), ipnumber)); + ipnumber += 1; + } + for (pub_key, num) in keys_nums { + let (seg1, seg2, seg3, seg4) = make_segments(num); + assert_eq!( + &subject + .node_by_key(&pub_key) + .unwrap() + .clone() + .metadata + .node_addr_opt, + &Some(NodeAddr::new( + &make_segmented_ip(seg1, seg2, seg3, seg4), + &[num] + )) + ); + } + } + #[test] fn add_half_neighbor_works() { let this_node = make_node_record(1234, true); @@ -756,19 +840,19 @@ mod tests { assert_eq!(result.matches("->").count(), 8); assert_string_contains( &result, - "\"AQIDBA\" [label=\"AR v1\\nAQIDBA\\n1.2.3.4:1234\"] [style=filled];", + "\"AQIDBA\" [label=\"AR v1 AD\\nAQIDBA\\n1.2.3.4:1234\"] [style=filled];", ); assert_string_contains( &result, - "\"AgMEBQ\" [label=\"AR v0\\nAgMEBQ\\n2.3.4.5:2345\"];", + "\"AgMEBQ\" [label=\"AR v0 AO\\nAgMEBQ\\n2.3.4.5:2345\"];", ); assert_string_contains( &result, - "\"AwQFBg\" [label=\"AR v0\\nAwQFBg\\n3.4.5.6:3456\"];", + "\"AwQFBg\" [label=\"AR v0 AO\\nAwQFBg\\n3.4.5.6:3456\"];", ); assert_string_contains( &result, - "\"BAUGBw\" [label=\"AR v0\\nBAUGBw\\n4.5.6.7:4567\"];", + "\"BAUGBw\" [label=\"AR v0 AO\\nBAUGBw\\n4.5.6.7:4567\"];", ); assert_string_contains(&result, "\"AQIDBA\" -> \"AgMEBQ\";"); assert_string_contains(&result, "\"AgMEBQ\" -> \"AQIDBA\";"); @@ -783,7 +867,13 @@ mod tests { #[test] fn new_public_ip_replaces_ip_address_and_nothing_else() { let this_node = make_node_record(1234, true); - let old_node = this_node.clone(); + let mut old_node = this_node.clone(); + old_node.inner.country_code_opt = Some("AD".to_string()); + old_node.metadata.node_location_opt = Some(NodeLocation { + country_code: "AD".to_string(), + }); + old_node.resign(); + let mut subject = NeighborhoodDatabase::new( this_node.public_key(), (&this_node).into(), @@ -794,6 +884,10 @@ mod tests { subject.new_public_ip(new_public_ip); + let this_pubkey = this_node.public_key(); + let updated_record = subject.node_by_key(this_pubkey).unwrap(); + old_node.metadata.last_update = updated_record.metadata.last_update; + let mut new_node = subject.root().clone(); assert_eq!(subject.node_by_ip(&new_public_ip), Some(&new_node)); assert_eq!( diff --git a/node/src/neighborhood/node_location.rs b/node/src/neighborhood/node_location.rs new file mode 100644 index 000000000..2b061d916 --- /dev/null +++ b/node/src/neighborhood/node_location.rs @@ -0,0 +1,49 @@ +// Copyright (c) 2024, MASQ (https://masq.ai) and/or its affiliates. All rights reserved. + +use ip_country_lib; +use ip_country_lib::country_finder::{CountryCodeFinder, COUNTRY_CODE_FINDER}; +use std::net::IpAddr; + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct NodeLocation { + pub country_code: String, +} + +pub fn get_node_location(ip_opt: Option) -> Option { + match ip_opt { + Some(ip_addr) => { + let country_opt = CountryCodeFinder::find_country(&COUNTRY_CODE_FINDER, ip_addr); + country_opt.map(|country| NodeLocation { + country_code: country.iso3166.clone(), + }) + } + None => None, + } +} + +#[cfg(test)] +mod tests { + use crate::neighborhood::node_location::{get_node_location, NodeLocation}; + use crate::neighborhood::node_record::NodeRecordMetadata; + use std::net::{IpAddr, Ipv4Addr}; + + #[test] + fn test_node_location() { + let node_location = get_node_location(Some(IpAddr::V4(Ipv4Addr::new(1, 2, 3, 4)))).unwrap(); + + assert_eq!(node_location.country_code, "AD"); + } + + #[test] + fn construct_node_record_metadata_with_free_world_bit() { + //TODO check in From impl for AGR that construction of metadata contains proper country_code and fwb, then delete this test + let mut metadata = NodeRecordMetadata::new(); + metadata.node_location_opt = get_node_location(Some(IpAddr::V4(Ipv4Addr::new(1, 2, 3, 4)))); + assert_eq!( + metadata.node_location_opt.as_ref().unwrap(), + &NodeLocation { + country_code: "AD".to_string(), + } + ); + } +} diff --git a/node/src/neighborhood/node_record.rs b/node/src/neighborhood/node_record.rs index e490cec1d..a15628ff5 100644 --- a/node/src/neighborhood/node_record.rs +++ b/node/src/neighborhood/node_record.rs @@ -1,8 +1,10 @@ // Copyright (c) 2019, MASQ (https://masq.ai) and/or its affiliates. All rights reserved. -use crate::neighborhood::gossip::GossipNodeRecord; +use crate::neighborhood::gossip::{ + regenerate_signed_gossip, AccessibleGossipRecord, GossipNodeRecord, +}; use crate::neighborhood::neighborhood_database::{NeighborhoodDatabase, NeighborhoodDatabaseError}; -use crate::neighborhood::{regenerate_signed_gossip, AccessibleGossipRecord}; +use crate::neighborhood::node_location::{get_node_location, NodeLocation}; use crate::sub_lib::cryptde::{CryptDE, CryptData, PlainData, PublicKey}; use crate::sub_lib::neighborhood::{NodeDescriptor, RatePack}; use crate::sub_lib::node_addr::NodeAddr; @@ -14,6 +16,7 @@ use std::collections::btree_set::BTreeSet; use std::collections::HashSet; use std::convert::TryFrom; +//TODO #584 create special serializer for NodeRecordInner_0v1 #[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] #[allow(non_camel_case_types)] pub struct NodeRecordInner_0v1 { @@ -24,6 +27,7 @@ pub struct NodeRecordInner_0v1 { pub accepts_connections: bool, pub routes_data: bool, pub version: u32, + pub country_code_opt: Option, } impl TryFrom for NodeRecordInner_0v1 { @@ -58,30 +62,42 @@ pub struct NodeRecord { pub signature: CryptData, } +#[derive(Clone)] +pub struct NodeRecordInputs { + pub earning_wallet: Wallet, + pub rate_pack: RatePack, + pub accepts_connections: bool, + pub routes_data: bool, + pub version: u32, + pub location_opt: Option, +} + impl NodeRecord { pub fn new( public_key: &PublicKey, - earning_wallet: Wallet, - rate_pack: RatePack, - accepts_connections: bool, - routes_data: bool, - version: u32, cryptde: &dyn CryptDE, // Must be the new NodeRecord's CryptDE: used for signing + node_record_inputs: NodeRecordInputs, ) -> NodeRecord { + let country_opt = node_record_inputs + .location_opt + .as_ref() + .map(|node_location| node_location.country_code.clone()); let mut node_record = NodeRecord { metadata: NodeRecordMetadata::new(), inner: NodeRecordInner_0v1 { public_key: public_key.clone(), - earning_wallet, - rate_pack, - accepts_connections, - routes_data, + earning_wallet: node_record_inputs.earning_wallet, + rate_pack: node_record_inputs.rate_pack, + accepts_connections: node_record_inputs.accepts_connections, + routes_data: node_record_inputs.routes_data, neighbors: BTreeSet::new(), - version, + version: node_record_inputs.version, + country_code_opt: country_opt, }, signed_gossip: PlainData::new(&[]), signature: CryptData::new(&[]), }; + node_record.metadata.node_location_opt = node_record_inputs.location_opt; node_record.regenerate_signed_gossip(cryptde); node_record } @@ -281,12 +297,17 @@ impl NodeRecord { impl From for NodeRecord { fn from(agr: AccessibleGossipRecord) -> Self { + let ip_add_opt = agr + .node_addr_opt + .as_ref() + .map(|node_rec| node_rec.ip_addr()); let mut node_record = NodeRecord { inner: agr.inner, metadata: NodeRecordMetadata::new(), signed_gossip: agr.signed_gossip, signature: agr.signature, }; + node_record.metadata.node_location_opt = get_node_location(ip_add_opt); node_record.metadata.node_addr_opt = agr.node_addr_opt; node_record } @@ -304,12 +325,17 @@ impl TryFrom<&GossipNodeRecord> for NodeRecord { fn try_from(gnr: &GossipNodeRecord) -> Result { let inner = NodeRecordInner_0v1::try_from(gnr)?; + let ip_addr_opt = gnr + .node_addr_opt + .as_ref() + .map(|node_rec| node_rec.ip_addr()); let mut node_record = NodeRecord { inner, metadata: NodeRecordMetadata::new(), signed_gossip: gnr.signed_data.clone(), signature: gnr.signature.clone(), }; + node_record.metadata.node_location_opt = get_node_location(ip_addr_opt); node_record.metadata.node_addr_opt = gnr.node_addr_opt.clone(); Ok(node_record) } @@ -321,6 +347,13 @@ pub struct NodeRecordMetadata { pub last_update: u32, pub node_addr_opt: Option, pub unreachable_hosts: HashSet, + pub node_location_opt: Option, + // country_undesirability is used in combination with FallbackRouting. If FallbackRouting is set + // to false, we do not consider the undesirability of countries other than those selected for exit. + // Therefore, we use a value of 0 for exit nodes in countries that are not considered for exit. + pub country_undesirability: u32, + //TODO introduce scores for latency #582 and reliability #583 + //TODO introduce check for node_location_opt, to verify full neighbors country code (we know his IP, so we can verify it) } impl NodeRecordMetadata { @@ -329,6 +362,8 @@ impl NodeRecordMetadata { last_update: time_t_timestamp(), node_addr_opt: None, unreachable_hosts: Default::default(), + node_location_opt: None, + country_undesirability: 0u32, } } } @@ -346,6 +381,21 @@ mod tests { use std::net::IpAddr; use std::str::FromStr; + #[test] + fn can_create_node_record_with_node_location_opt_none() { + let mut node_record_wo_location = make_node_record(2222, false); + node_record_wo_location.inner.accepts_connections = false; + let no_location_db = db_from_node(&node_record_wo_location); + let no_location_gossip = GossipBuilder::new(&no_location_db) + .node(node_record_wo_location.public_key(), false) + .build(); + + let nr_wo_location = + NodeRecord::try_from(no_location_gossip.node_records.first().unwrap()).unwrap(); + + assert_eq!(nr_wo_location.inner.country_code_opt, None); + } + #[test] fn can_create_a_node_record_from_a_reference() { let mut expected_node_record = make_node_record(1234, true); @@ -354,11 +404,22 @@ mod tests { let mut db = db_from_node(&make_node_record(2345, true)); db.add_node(expected_node_record.clone()).unwrap(); let builder = GossipBuilder::new(&db).node(expected_node_record.public_key(), true); + let before = time_t_timestamp(); let actual_node_record = NodeRecord::try_from(builder.build().node_records.first().unwrap()).unwrap(); - assert_eq!(expected_node_record, actual_node_record); + let after = time_t_timestamp(); + assert!( + before <= actual_node_record.metadata.last_update + && actual_node_record.metadata.last_update <= after + ); + assert_eq!( + actual_node_record.inner.country_code_opt, + Some("AD".to_string()) + ); + expected_node_record.metadata.last_update = actual_node_record.metadata.last_update; + assert_eq!(actual_node_record, expected_node_record); } #[test] @@ -583,53 +644,41 @@ mod tests { #[test] fn node_record_partial_eq() { let earning_wallet = make_wallet("wallet"); + let node_record_data = NodeRecordInputs { + earning_wallet: earning_wallet.clone(), + rate_pack: rate_pack(100), + accepts_connections: true, + routes_data: true, + version: 0, + location_opt: None, + }; let exemplar = NodeRecord::new( &PublicKey::new(&b"poke"[..]), - earning_wallet.clone(), - rate_pack(100), - true, - true, - 0, main_cryptde(), + node_record_data.clone(), ); let duplicate = NodeRecord::new( &PublicKey::new(&b"poke"[..]), - earning_wallet.clone(), - rate_pack(100), - true, - true, - 0, main_cryptde(), + node_record_data.clone(), ); let mut with_neighbor = NodeRecord::new( &PublicKey::new(&b"poke"[..]), - earning_wallet.clone(), - rate_pack(100), - true, - true, - 0, main_cryptde(), + node_record_data.clone(), ); let mod_key = NodeRecord::new( &PublicKey::new(&b"kope"[..]), - earning_wallet.clone(), - rate_pack(100), - true, - true, - 0, main_cryptde(), + node_record_data.clone(), ); with_neighbor .add_half_neighbor_key(mod_key.public_key().clone()) .unwrap(); let mut mod_node_addr = NodeRecord::new( &PublicKey::new(&b"poke"[..]), - earning_wallet.clone(), - rate_pack(100), - true, - true, - 0, main_cryptde(), + node_record_data.clone(), ); mod_node_addr .set_node_addr(&NodeAddr::new( @@ -637,70 +686,52 @@ mod tests { &[1234], )) .unwrap(); + let mut node_record_data_mod_earning_wallet = node_record_data.clone(); + node_record_data_mod_earning_wallet.earning_wallet = make_wallet("booga"); let mod_earning_wallet = NodeRecord::new( &PublicKey::new(&b"poke"[..]), - make_wallet("booga"), - rate_pack(100), - true, - true, - 0, main_cryptde(), + node_record_data_mod_earning_wallet, ); + let mut node_record_data_mod_rate_pack = node_record_data.clone(); + node_record_data_mod_rate_pack.rate_pack = rate_pack(200); let mod_rate_pack = NodeRecord::new( &PublicKey::new(&b"poke"[..]), - earning_wallet.clone(), - rate_pack(200), - true, - true, - 0, main_cryptde(), + node_record_data_mod_rate_pack, ); + let mut node_record_data_mod_accepts_connections = node_record_data.clone(); + node_record_data_mod_accepts_connections.accepts_connections = false; let mod_accepts_connections = NodeRecord::new( &PublicKey::new(&b"poke"[..]), - earning_wallet.clone(), - rate_pack(100), - false, - true, - 0, main_cryptde(), + node_record_data_mod_accepts_connections, ); + let mut node_record_data_mod_routes_data = node_record_data.clone(); + node_record_data_mod_routes_data.routes_data = false; let mod_routes_data = NodeRecord::new( &PublicKey::new(&b"poke"[..]), - earning_wallet.clone(), - rate_pack(100), - true, - false, - 0, main_cryptde(), + node_record_data_mod_routes_data, ); let mut mod_signed_gossip = NodeRecord::new( &PublicKey::new(&b"poke"[..]), - earning_wallet.clone(), - rate_pack(100), - true, - true, - 0, main_cryptde(), + node_record_data.clone(), ); mod_signed_gossip.signed_gossip = mod_rate_pack.signed_gossip.clone(); let mut mod_signature = NodeRecord::new( &PublicKey::new(&b"poke"[..]), - earning_wallet.clone(), - rate_pack(100), - true, - true, - 0, main_cryptde(), + node_record_data.clone(), ); mod_signature.signature = CryptData::new(&[]); + let mut node_record_data_mod_version = node_record_data.clone(); + node_record_data_mod_version.version = 1; let mod_version = NodeRecord::new( &PublicKey::new(&b"poke"[..]), - earning_wallet.clone(), - rate_pack(100), - true, - true, - 1, main_cryptde(), + node_record_data_mod_version, ); assert_eq!(exemplar, exemplar); diff --git a/node/src/node_configurator/node_configurator_standard.rs b/node/src/node_configurator/node_configurator_standard.rs index a69e7b527..79e65aff5 100644 --- a/node/src/node_configurator/node_configurator_standard.rs +++ b/node/src/node_configurator/node_configurator_standard.rs @@ -1147,12 +1147,11 @@ mod tests { } #[test] - fn server_initializer_collected_params_handle_config_file_from_environment_and_real_user_from_config_file_with_data_directory( - ) { + fn config_file_from_env_and_real_user_from_config_file_with_data_directory_from_command_line() { running_test(); let _guard = EnvironmentGuard::new(); let _clap_guard = ClapGuard::new(); - let home_dir = ensure_node_home_directory_exists( "node_configurator_standard","server_initializer_collected_params_handle_config_file_from_environment_and_real_user_from_config_file_with_data_directory"); + let home_dir = ensure_node_home_directory_exists( "node_configurator_standard","config_file_from_env_and_real_user_from_config_file_with_data_directory_from_command_line"); let data_dir = &home_dir.join("data_dir"); create_dir_all(home_dir.join("config")).expect("expected directory for config"); let config_file_relative = File::create(&home_dir.join("config/config.toml")).unwrap(); diff --git a/node/src/proxy_server/mod.rs b/node/src/proxy_server/mod.rs index 372b52c37..077b1bfe5 100644 --- a/node/src/proxy_server/mod.rs +++ b/node/src/proxy_server/mod.rs @@ -2217,7 +2217,7 @@ mod tests { target_component: Component::ProxyClient, return_component_opt: Some(Component::ProxyServer), payload_size: 47, - hostname_opt: Some("nowhere.com".to_string()) + hostname_opt: Some("nowhere.com".to_string()), } ); let dispatcher_recording = dispatcher_log_arc.lock().unwrap(); @@ -2297,7 +2297,7 @@ mod tests { target_component: Component::ProxyClient, return_component_opt: Some(Component::ProxyServer), payload_size: 16, - hostname_opt: None + hostname_opt: None, } ); let dispatcher_recording = dispatcher_log_arc.lock().unwrap(); diff --git a/node/src/server_initializer.rs b/node/src/server_initializer.rs index ea288fdce..54e8b0b36 100644 --- a/node/src/server_initializer.rs +++ b/node/src/server_initializer.rs @@ -397,14 +397,13 @@ pub mod tests { use masq_lib::crash_point::CrashPoint; use masq_lib::multi_config::MultiConfig; use masq_lib::shared_schema::{ConfiguratorError, ParamError}; - use masq_lib::test_utils::fake_stream_holder::{ - ByteArrayReader, ByteArrayWriter, FakeStreamHolder, - }; + use masq_lib::test_utils::fake_stream_holder::FakeStreamHolder; use masq_lib::test_utils::logging::{init_test_logging, TestLogHandler}; use masq_lib::utils::slice_of_strs_to_vec_of_strings; use std::cell::RefCell; use std::ops::Not; use std::sync::{Arc, Mutex}; + use test_utilities::byte_array_reader_writer::{ByteArrayReader, ByteArrayWriter}; impl ConfiguredByPrivilege for CrashTestDummy { fn initialize_as_privileged( diff --git a/node/src/sub_lib/migrations/node_record_inner.rs b/node/src/sub_lib/migrations/node_record_inner.rs index 7d9616c98..be978f8bf 100644 --- a/node/src/sub_lib/migrations/node_record_inner.rs +++ b/node/src/sub_lib/migrations/node_record_inner.rs @@ -55,6 +55,7 @@ impl TryFrom<&Value> for NodeRecordInner_0v1 { let mut accepts_connections_opt: Option = None; let mut routes_data_opt: Option = None; let mut version_opt: Option = None; + let mut country_code_opt: Option = None; map.keys().for_each(|k| { let v = map.get(k).expect("Disappeared"); match (k, v) { @@ -96,6 +97,12 @@ impl TryFrom<&Value> for NodeRecordInner_0v1 { _ => (), } } + (Value::Text(field_name), Value::Text(field_value)) => { + match field_name.as_str() { + "country_code" => country_code_opt = Some(field_value.clone()), + _ => (), + } + } _ => (), } }); @@ -120,6 +127,7 @@ impl TryFrom<&Value> for NodeRecordInner_0v1 { ); check_field(&mut missing_fields, "routes_data", &routes_data_opt); check_field(&mut missing_fields, "version", &version_opt); + check_field(&mut missing_fields, "country_code", &country_code_opt); if !missing_fields.is_empty() { unimplemented!("{:?}", missing_fields.clone()) } @@ -131,6 +139,7 @@ impl TryFrom<&Value> for NodeRecordInner_0v1 { accepts_connections: accepts_connections_opt.expect("public_key disappeared"), routes_data: routes_data_opt.expect("public_key disappeared"), version: version_opt.expect("public_key disappeared"), + country_code_opt, }) } _ => Err(StepError::SemanticError(format!( @@ -173,6 +182,7 @@ mod tests { pub accepts_connections: bool, pub routes_data: bool, pub version: u32, + pub country_code: Option, pub another_field: String, pub yet_another_field: u64, } @@ -186,6 +196,7 @@ mod tests { accepts_connections: false, routes_data: true, version: 42, + country_code_opt: Some("AU".to_string()), }; let future_nri = ExampleFutureNRI { public_key: expected_nri.public_key.clone(), @@ -195,6 +206,7 @@ mod tests { accepts_connections: expected_nri.accepts_connections, routes_data: expected_nri.routes_data, version: expected_nri.version, + country_code: expected_nri.country_code_opt.clone(), another_field: "These are the times that try men's souls".to_string(), yet_another_field: 1234567890, }; diff --git a/node/src/sub_lib/neighborhood.rs b/node/src/sub_lib/neighborhood.rs index d3acc5655..79623cda3 100644 --- a/node/src/sub_lib/neighborhood.rs +++ b/node/src/sub_lib/neighborhood.rs @@ -3,7 +3,7 @@ use crate::neighborhood::gossip::Gossip_0v1; use crate::neighborhood::node_record::NodeRecord; use crate::neighborhood::overall_connection_status::ConnectionProgress; -use crate::neighborhood::Neighborhood; +use crate::neighborhood::{Neighborhood, UserExitPreferences}; use crate::sub_lib::cryptde::{CryptDE, PublicKey}; use crate::sub_lib::cryptde_real::CryptDEReal; use crate::sub_lib::dispatcher::{Component, StreamShutdownMsg}; @@ -244,7 +244,7 @@ impl NodeDescriptor { CHAIN_IDENTIFIER_DELIMITER, contact_public_key_string, CENTRAL_DELIMITER, - node_addr_string + node_addr_string, ) } @@ -598,6 +598,7 @@ pub struct NeighborhoodMetadata { pub connection_progress_peers: Vec, pub cpm_recipient: Recipient, pub db_patch_size: u8, + pub user_exit_preferences_opt: Option, } pub struct NeighborhoodTools { @@ -1060,7 +1061,7 @@ mod tests { target_component: Component::ProxyClient, return_component_opt: Some(Component::ProxyServer), payload_size: 7500, - hostname_opt: None + hostname_opt: None, } ); } diff --git a/node/src/test_utils/neighborhood_test_utils.rs b/node/src/test_utils/neighborhood_test_utils.rs index 25dfeeadb..290b905c0 100644 --- a/node/src/test_utils/neighborhood_test_utils.rs +++ b/node/src/test_utils/neighborhood_test_utils.rs @@ -1,9 +1,12 @@ // Copyright (c) 2019, MASQ (https://masq.ai) and/or its affiliates. All rights reserved. use crate::bootstrapper::BootstrapperConfig; -use crate::neighborhood::gossip::{GossipBuilder, GossipNodeRecord, Gossip_0v1}; +use crate::neighborhood::gossip::{ + AccessibleGossipRecord, GossipBuilder, GossipNodeRecord, Gossip_0v1, +}; use crate::neighborhood::neighborhood_database::NeighborhoodDatabase; -use crate::neighborhood::node_record::{NodeRecord, NodeRecordInner_0v1}; -use crate::neighborhood::{AccessibleGossipRecord, Neighborhood, DEFAULT_MIN_HOPS}; +use crate::neighborhood::node_location::NodeLocation; +use crate::neighborhood::node_record::{NodeRecord, NodeRecordInner_0v1, NodeRecordInputs}; +use crate::neighborhood::{Neighborhood, DEFAULT_MIN_HOPS}; use crate::sub_lib::cryptde::PublicKey; use crate::sub_lib::cryptde::{CryptDE, PlainData}; use crate::sub_lib::cryptde_null::CryptDENull; @@ -12,6 +15,7 @@ use crate::sub_lib::node_addr::NodeAddr; use crate::sub_lib::wallet::Wallet; use crate::test_utils::*; use ethereum_types::H160; +use ip_country_lib::country_finder::COUNTRY_CODE_FINDER; use masq_lib::blockchains::chains::Chain; use masq_lib::test_utils::utils::TEST_DEFAULT_CHAIN; use std::convert::TryFrom; @@ -21,6 +25,20 @@ use std::net::Ipv4Addr; pub const MIN_HOPS_FOR_TEST: Hops = DEFAULT_MIN_HOPS; pub const DB_PATCH_SIZE_FOR_TEST: u8 = DEFAULT_MIN_HOPS as u8; +// lazy_static! { +// pub static ref COUNTRY_CODE_DIGEST: Vec<(IpAddr, String)> = vec![ +// ( +// IpAddr::V4(Ipv4Addr::new(123, 123, 123, 123)), +// "CN".to_string(), +// ), +// (IpAddr::V4(Ipv4Addr::new(0, 0, 0, 123)), "US".to_string(),), +// (IpAddr::V4(Ipv4Addr::new(99, 99, 99, 99)), "FR".to_string(),), +// (IpAddr::V4(Ipv4Addr::new(3, 3, 3, 3)), "AU".to_string(),), +// (IpAddr::V4(Ipv4Addr::new(101, 0, 0, 255)), "AU".to_string(),), +// (IpAddr::V4(Ipv4Addr::new(255, 0, 0, 220)), "FR".to_string(),), +// ]; +// } + impl From<(&NeighborhoodDatabase, &PublicKey, bool)> for AccessibleGossipRecord { fn from( (database, public_key, reveal_node_addr): (&NeighborhoodDatabase, &PublicKey, bool), @@ -30,14 +48,30 @@ impl From<(&NeighborhoodDatabase, &PublicKey, bool)> for AccessibleGossipRecord } } -pub fn make_node_record(n: u16, has_ip: bool) -> NodeRecord { +pub fn make_segments(n: u16) -> (u8, u8, u8, u8) { let seg1 = ((n / 1000) % 10) as u8; let seg2 = ((n / 100) % 10) as u8; let seg3 = ((n / 10) % 10) as u8; let seg4 = (n % 10) as u8; + (seg1, seg2, seg3, seg4) +} + +pub fn make_segmented_ip(seg1: u8, seg2: u8, seg3: u8, seg4: u8) -> IpAddr { + IpAddr::V4(Ipv4Addr::new(seg1, seg2, seg3, seg4)) +} + +pub fn make_node_record(n: u16, has_ip: bool) -> NodeRecord { + let (seg1, seg2, seg3, seg4) = make_segments(n); let key = PublicKey::new(&[seg1, seg2, seg3, seg4]); - let ip_addr = IpAddr::V4(Ipv4Addr::new(seg1, seg2, seg3, seg4)); + let ip_addr = make_segmented_ip(seg1, seg2, seg3, seg4); let node_addr = NodeAddr::new(&ip_addr, &[n % 10000]); + let country_opt = COUNTRY_CODE_FINDER.find_country(ip_addr); + let location_opt = match has_ip { + true => country_opt.map(|country| NodeLocation { + country_code: country.iso3166.clone(), + }), + false => None, + }; NodeRecord::new_for_tests( &key, @@ -45,6 +79,7 @@ pub fn make_node_record(n: u16, has_ip: bool) -> NodeRecord { u64::from(n), true, true, + location_opt, ) } @@ -156,15 +191,20 @@ impl NodeRecord { base_rate: u64, accepts_connections: bool, routes_data: bool, + node_location_opt: Option, ) -> NodeRecord { - let mut node_record = NodeRecord::new( - public_key, - NodeRecord::earning_wallet_from_key(public_key), - rate_pack(base_rate), + let node_record_data = NodeRecordInputs { + earning_wallet: NodeRecord::earning_wallet_from_key(public_key), + rate_pack: rate_pack(base_rate), accepts_connections, routes_data, - 0, + version: 0, + location_opt: node_location_opt, + }; + let mut node_record = NodeRecord::new( + public_key, &CryptDENull::from(public_key, TEST_DEFAULT_CHAIN), + node_record_data, ); if let Some(node_addr) = node_addr_opt { node_record.set_node_addr(node_addr).unwrap(); diff --git a/node/tests/initialization_test.rs b/node/tests/initialization_test.rs index 3ea97ab04..5b57d5b96 100644 --- a/node/tests/initialization_test.rs +++ b/node/tests/initialization_test.rs @@ -260,7 +260,7 @@ fn requested_chain_meets_different_db_chain_and_panics_integration() { r"ERROR: PanicHandler: src(/|\\)actor_system_factory\.rs.*- Database with a wrong chain name detected; expected: {}, was: eth-mainnet", &chain_literal ); - node.wait_for_log(®ex_pattern, Some(1000)); + node.wait_for_log(®ex_pattern, Some(5000)); } #[test] diff --git a/node/tests/ui_gateway_test.rs b/node/tests/ui_gateway_test.rs index 042cfda42..eb3658f9c 100644 --- a/node/tests/ui_gateway_test.rs +++ b/node/tests/ui_gateway_test.rs @@ -13,6 +13,10 @@ use masq_lib::messages::{ use masq_lib::test_utils::ui_connection::UiConnection; use masq_lib::test_utils::utils::ensure_node_home_directory_exists; use masq_lib::utils::{add_chain_specific_directory, find_free_port}; +use std::net::TcpStream; +use std::thread; +use std::time::{Duration, SystemTime}; +use sysinfo::{ProcessExt, System, SystemExt}; use utils::CommandConfig; #[test] @@ -54,6 +58,7 @@ fn ui_requests_something_and_gets_corresponding_response() { #[test] fn log_broadcasts_are_correctly_received_integration() { + wait_for_masq_node_ends(); fdlimit::raise_fd_limit(); let port = find_free_port(); let mut node = utils::MASQNode::start_standard( @@ -92,11 +97,32 @@ fn log_broadcasts_are_correctly_received_integration() { node.wait_for_exit(); } +fn wait_for_masq_node_ends() { + let mut system = System::new_all(); + let deadline = SystemTime::now() + Duration::from_secs(5); + loop { + if SystemTime::now() > deadline { + panic!("Previous instance of MASQNode does not stops"); + } + system.refresh_all(); + if system + .processes() + .into_iter() + .find(|(_, process)| process.name().contains("MASQNode")) + .is_none() + { + break; + } + thread::sleep(Duration::from_millis(500)); + } +} + #[test] fn daemon_does_not_allow_node_to_keep_his_client_alive_integration() { //Daemon's probe to check if the Node is alive causes an unwanted new reference //for the Daemon's client, so we need to make the Daemon send a close message //breaking any reference to him immediately + wait_for_masq_node_ends(); fdlimit::raise_fd_limit(); let data_directory = ensure_node_home_directory_exists( "ui_gateway_test", @@ -163,6 +189,13 @@ fn daemon_does_not_allow_node_to_keep_his_client_alive_integration() { let assertion_lookup_pattern_2 = |_port_spec_ui: &str| "Received shutdown order from client 1".to_string(); let second_port = connected_and_disconnected_assertion(2, assertion_lookup_pattern_2); + //TODO Card #806 "Test utility to easily verify the Node's termination" + loop { + if let Ok(_stream) = TcpStream::connect(format!("127.0.0.1:{}", ui_redirect.port)) { + } else { + break; + } + } let _ = daemon.kill(); daemon.wait_for_exit(); //only an additional assertion checking the involved clients to have different port numbers @@ -171,6 +204,7 @@ fn daemon_does_not_allow_node_to_keep_his_client_alive_integration() { #[test] fn cleanup_after_deceased_clients_integration() { + wait_for_masq_node_ends(); fdlimit::raise_fd_limit(); let port = find_free_port(); let mut node = utils::MASQNode::start_standard( diff --git a/node/tests/utils.rs b/node/tests/utils.rs index adeec8c7e..22c258e9b 100644 --- a/node/tests/utils.rs +++ b/node/tests/utils.rs @@ -3,8 +3,10 @@ use itertools::Itertools; use masq_lib::blockchains::chains::Chain; use masq_lib::constants::{CURRENT_LOGFILE_NAME, DEFAULT_CHAIN, DEFAULT_UI_PORT}; -use masq_lib::test_utils::utils::{ensure_node_home_directory_exists, node_home_directory}; -use masq_lib::utils::{add_masq_and_chain_directories, localhost}; +use masq_lib::test_utils::utils::{ + ensure_node_home_directory_exists, node_home_directory, recreate_data_dir, +}; +use masq_lib::utils::{add_masq_and_chain_directories, localhost, running_test}; use node_lib::database::db_initializer::{ DbInitializationConfig, DbInitializer, DbInitializerReal, }; @@ -359,10 +361,15 @@ impl MASQNode { ensure_start: bool, command_getter: F, ) -> MASQNode { - let data_dir = if sterile_database { - ensure_node_home_directory_exists("integration", test_name) - } else { - node_home_directory("integration", test_name) + running_test(); + let data_dir = match ( + sterile_database, + Self::data_directory_from_config_opt(&config_opt), + ) { + (true, None) => ensure_node_home_directory_exists("integration", test_name), + (false, None) => node_home_directory("integration", test_name), + (false, Some(conf_data_dir)) => PathBuf::from(conf_data_dir), + (true, Some(data_dir)) => recreate_data_dir(&PathBuf::from(data_dir)), }; if sterile_logfile { let _ = Self::remove_logfile(&data_dir); @@ -642,6 +649,13 @@ impl MASQNode { }, } } + + fn data_directory_from_config_opt(config_opt: &Option) -> Option { + match config_opt { + None => None, + Some(config) => config.value_of("--data-directory"), + } + } } #[derive(Debug)] diff --git a/test_utilities/Cargo.toml b/test_utilities/Cargo.toml new file mode 100644 index 000000000..93191e691 --- /dev/null +++ b/test_utilities/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "test_utilities" +version = "0.1.0" +edition = "2021" +authors = ["Dan Wiebe ", "MASQ"] +license = "GPL-3.0-only" +description = "Testing utilities Code common to Node and masq; also, temporarily, to dns_utility" +workspace = "../node" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] \ No newline at end of file diff --git a/test_utilities/src/byte_array_reader_writer.rs b/test_utilities/src/byte_array_reader_writer.rs new file mode 100644 index 000000000..fe611f781 --- /dev/null +++ b/test_utilities/src/byte_array_reader_writer.rs @@ -0,0 +1,132 @@ +// Copyright (c) 2019, MASQ (https://masq.ai) and/or its affiliates. All rights reserved. + +use std::cmp::min; +use std::io; +use std::io::Read; +use std::io::Write; +use std::io::{BufRead, Error}; +use std::sync::{Arc, Mutex}; + +pub struct ByteArrayWriter { + inner_arc: Arc>, +} + +pub struct ByteArrayWriterInner { + byte_array: Vec, + next_error: Option, +} + +impl ByteArrayWriterInner { + pub fn get_bytes(&self) -> Vec { + self.byte_array.clone() + } + pub fn get_string(&self) -> String { + String::from_utf8(self.get_bytes()).unwrap() + } +} + +impl Default for ByteArrayWriter { + fn default() -> Self { + ByteArrayWriter { + inner_arc: Arc::new(Mutex::new(ByteArrayWriterInner { + byte_array: vec![], + next_error: None, + })), + } + } +} + +impl ByteArrayWriter { + pub fn new() -> ByteArrayWriter { + Self::default() + } + + pub fn inner_arc(&self) -> Arc> { + self.inner_arc.clone() + } + + pub fn get_bytes(&self) -> Vec { + self.inner_arc.lock().unwrap().byte_array.clone() + } + pub fn get_string(&self) -> String { + String::from_utf8(self.get_bytes()).unwrap() + } + + pub fn reject_next_write(&mut self, error: Error) { + self.inner_arc().lock().unwrap().next_error = Some(error); + } +} + +impl Write for ByteArrayWriter { + fn write(&mut self, buf: &[u8]) -> io::Result { + let mut inner = self.inner_arc.lock().unwrap(); + if let Some(next_error) = inner.next_error.take() { + Err(next_error) + } else { + for byte in buf { + inner.byte_array.push(*byte) + } + Ok(buf.len()) + } + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +pub struct ByteArrayReader { + byte_array: Vec, + position: usize, + next_error: Option, +} + +impl ByteArrayReader { + pub fn new(byte_array: &[u8]) -> ByteArrayReader { + ByteArrayReader { + byte_array: byte_array.to_vec(), + position: 0, + next_error: None, + } + } + + pub fn reject_next_read(mut self, error: Error) -> ByteArrayReader { + self.next_error = Some(error); + self + } +} + +impl Read for ByteArrayReader { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + match self.next_error.take() { + Some(error) => Err(error), + None => { + let to_copy = min(buf.len(), self.byte_array.len() - self.position); + #[allow(clippy::needless_range_loop)] + for idx in 0..to_copy { + buf[idx] = self.byte_array[self.position + idx] + } + self.position += to_copy; + Ok(to_copy) + } + } + } +} + +impl BufRead for ByteArrayReader { + fn fill_buf(&mut self) -> io::Result<&[u8]> { + match self.next_error.take() { + Some(error) => Err(error), + None => Ok(&self.byte_array[self.position..]), + } + } + + fn consume(&mut self, amt: usize) { + let result = self.position + amt; + self.position = if result < self.byte_array.len() { + result + } else { + self.byte_array.len() + } + } +} diff --git a/test_utilities/src/lib.rs b/test_utilities/src/lib.rs new file mode 100644 index 000000000..552c3b7d1 --- /dev/null +++ b/test_utilities/src/lib.rs @@ -0,0 +1,3 @@ +// Copyright (c) 2024, MASQ (https://masq.ai) and/or its affiliates. All rights reserved. + +pub mod byte_array_reader_writer;