Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jp/p3 client 20230619 #437

Draft
wants to merge 30 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
14e26e5
p3_server_peer/p3_server_peers config
JamesPiechota Jun 19, 2023
c1e89af
log peer stats
JamesPiechota Jun 22, 2023
50218e3
logging around what is driving up some performance metrics
JamesPiechota Jun 22, 2023
ffc9918
rebalance sync peers every minute
JamesPiechota Jun 24, 2023
dd1aaf0
call update_rating on all web requests
JamesPiechota Jun 26, 2023
5e6eeee
better handling of the blacklist domain, also try to reduce
JamesPiechota Jun 26, 2023
0bce164
Don't track host-based peers (e.g. the blacklist URL)
JamesPiechota Jun 27, 2023
4e8b057
Simplify handling of ReadTime and ReadBodySize
JamesPiechota Jun 28, 2023
d3e28eb
remove erlang:put/get read_body_time and body_size
JamesPiechota Jun 28, 2023
a7f66fb
EMA for performance metrics
JamesPiechota Jun 29, 2023
8529714
format as ms
JamesPiechota Jun 29, 2023
68e6760
don't let gossiped data skew the throughput metrics
JamesPiechota Jun 30, 2023
cf17ed7
print the peer stats after ranking
JamesPiechota Jun 30, 2023
a2711af
penalized bad fetched data
JamesPiechota Jul 1, 2023
2163014
fixup
JamesPiechota Jul 1, 2023
9990792
track concurrent requests in ar_peers rating
Jul 2, 2023
93ab2f1
WIP
Jul 2, 2023
aaae82e
WIP
JamesPiechota Jul 4, 2023
1bb1989
WIP moving to {Peer, Metric, Performance} format
JamesPiechota Jul 5, 2023
acf4208
spaces to tabs
JamesPiechota Jul 5, 2023
c8c1a1b
WIP
JamesPiechota Jul 5, 2023
a6362aa
WIP
JamesPiechota Jul 5, 2023
9e45c2e
WIP
JamesPiechota Jul 6, 2023
b1ac4bc
call rate_xxx wherever we called send(peer (except for chunks)
JamesPiechota Jul 6, 2023
066218d
WIP
JamesPiechota Jul 7, 2023
a51d5e7
revert metric addition
JamesPiechota Jul 7, 2023
ab54e3b
Cleanup how we rate new blocks that are fetched vs. gossiped
JamesPiechota Jul 7, 2023
9a4185d
some last Timestamp -> QueryBlockTime changes
JamesPiechota Jul 7, 2023
0425e88
replace bad_response with issue_warning
JamesPiechota Jul 7, 2023
4e98f13
move ban logic into ar_peers
JamesPiechota Jul 7, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 0 additions & 2 deletions apps/arweave/include/ar.hrl
Expand Up @@ -7,8 +7,6 @@
%% (e.g. bin/test or bin/shell)
-define(IS_TEST, erlang:get_cookie() == test).

-define(DATA_SIZE(Term), erlang:byte_size(term_to_binary(Term))).

%% The mainnet name. Does not change at the hard forks.
-ifndef(NETWORK_NAME).
-ifdef(DEBUG).
Expand Down
1 change: 1 addition & 0 deletions apps/arweave/include/ar_config.hrl
Expand Up @@ -191,6 +191,7 @@
= ?DEFAULT_MAX_NONCE_LIMITER_LAST_STEP_VALIDATION_THREAD_COUNT,
nonce_limiter_server_trusted_peers = [],
nonce_limiter_client_peers = [],
p3_server_peers = [],
debug = false,
repair_rocksdb = [],
run_defragmentation = false,
Expand Down
21 changes: 21 additions & 0 deletions apps/arweave/include/ar_peers.hrl
@@ -0,0 +1,21 @@
-ifndef(AR_PEERS_HRL).
-define(AR_PEERS_HRL, true).

-include_lib("ar.hrl").

%% factor to scale the average throughput by when rating gossiped data - lower is better
-define(GOSSIP_ADVANTAGE, 0.5).

-record(performance, {
version = 3,
release = -1,
average_bytes = 0.0,
total_bytes = 0,
average_latency = 0.0,
total_latency = 0.0,
transfers = 0,
average_success = 1.0,
rating = 0
}).

-endif.
3 changes: 3 additions & 0 deletions apps/arweave/src/ar.erl
Expand Up @@ -541,6 +541,9 @@ parse_cli_args(["vdf_server_trusted_peer", Peer | Rest], C) ->
parse_cli_args(["vdf_client_peer", RawPeer | Rest],
C = #config{ nonce_limiter_client_peers = Peers }) ->
parse_cli_args(Rest, C#config{ nonce_limiter_client_peers = [RawPeer | Peers] });
parse_cli_args(["p3_server_peer", RawPeer | Rest],
C = #config{ p3_server_peers = Peers }) ->
parse_cli_args(Rest, C#config{ p3_server_peers = [RawPeer | Peers] });
parse_cli_args(["debug" | Rest], C) ->
parse_cli_args(Rest, C#config{ debug = true });
parse_cli_args(["repair_rocksdb", Path | Rest], #config{ repair_rocksdb = L } = C) ->
Expand Down
1 change: 0 additions & 1 deletion apps/arweave/src/ar_blacklist_middleware.erl
Expand Up @@ -45,7 +45,6 @@ start() ->
ban_peer(Peer, TTLSeconds) ->
Key = {ban, peer_to_ip_addr(Peer)},
Expires = os:system_time(seconds) + TTLSeconds,
ar_events:send(peer, {banned, Peer}),
ets:insert(?MODULE, {Key, Expires}).

is_peer_banned(Peer) ->
Expand Down