From e490eda3d3185d227758e49fc137910f3246e5da Mon Sep 17 00:00:00 2001 From: Santiago Lema Date: Wed, 21 May 2025 02:02:57 +0000 Subject: [PATCH 01/12] added .gitkeep to create _already_sent and _credentials folders --- _already_sent/.gitkeep | 0 _credentials/.gitkeep | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 _already_sent/.gitkeep create mode 100644 _credentials/.gitkeep diff --git a/_already_sent/.gitkeep b/_already_sent/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/_credentials/.gitkeep b/_credentials/.gitkeep new file mode 100644 index 00000000..e69de29b From 8dc8688d2e62da0d25106005478a994b6fa052fb Mon Sep 17 00:00:00 2001 From: Santiago Lema Date: Wed, 21 May 2025 02:11:08 +0000 Subject: [PATCH 02/12] Untrack files from _already_sent and keep folder with .gitkeep --- _already_sent/0cf7dedb7cfac4340056a612bc3c50b1.txt | 1 - _already_sent/12b2cc8c05f23758c6e812ef34044f53.txt | 1 - _already_sent/3f2721d52927f2d2c5f1146665b441dd.txt | 1 - _already_sent/8eef8d9550fb4952c4ef2ba2656b4038.txt | 1 - _already_sent/a3ffc5f64551046ad7132d159f1f40e7.txt | 1 - _already_sent/db41d26877002dfa9dba650122b8a298.txt | 1 - 6 files changed, 6 deletions(-) delete mode 100644 _already_sent/0cf7dedb7cfac4340056a612bc3c50b1.txt delete mode 100644 _already_sent/12b2cc8c05f23758c6e812ef34044f53.txt delete mode 100644 _already_sent/3f2721d52927f2d2c5f1146665b441dd.txt delete mode 100644 _already_sent/8eef8d9550fb4952c4ef2ba2656b4038.txt delete mode 100644 _already_sent/a3ffc5f64551046ad7132d159f1f40e7.txt delete mode 100644 _already_sent/db41d26877002dfa9dba650122b8a298.txt diff --git a/_already_sent/0cf7dedb7cfac4340056a612bc3c50b1.txt b/_already_sent/0cf7dedb7cfac4340056a612bc3c50b1.txt deleted file mode 100644 index a963820c..00000000 --- a/_already_sent/0cf7dedb7cfac4340056a612bc3c50b1.txt +++ /dev/null @@ -1 +0,0 @@ -https://www.openculture.com/2025/05/how-frank-lloyd-wrights-architecture-evolved-over-70-years-and-changed-america.html \ No newline at end of file diff --git a/_already_sent/12b2cc8c05f23758c6e812ef34044f53.txt b/_already_sent/12b2cc8c05f23758c6e812ef34044f53.txt deleted file mode 100644 index 83138f71..00000000 --- a/_already_sent/12b2cc8c05f23758c6e812ef34044f53.txt +++ /dev/null @@ -1 +0,0 @@ -https://en.wikipedia.org/wiki/Rainhill_trials \ No newline at end of file diff --git a/_already_sent/3f2721d52927f2d2c5f1146665b441dd.txt b/_already_sent/3f2721d52927f2d2c5f1146665b441dd.txt deleted file mode 100644 index e60c2b8a..00000000 --- a/_already_sent/3f2721d52927f2d2c5f1146665b441dd.txt +++ /dev/null @@ -1 +0,0 @@ -https://fossforce.com/2025/04/is-free-or-open-source-software-sustainable/ \ No newline at end of file diff --git a/_already_sent/8eef8d9550fb4952c4ef2ba2656b4038.txt b/_already_sent/8eef8d9550fb4952c4ef2ba2656b4038.txt deleted file mode 100644 index db0cfd4f..00000000 --- a/_already_sent/8eef8d9550fb4952c4ef2ba2656b4038.txt +++ /dev/null @@ -1 +0,0 @@ -https://manualdousuario.net/en/writing-chatgpt-ai/ \ No newline at end of file diff --git a/_already_sent/a3ffc5f64551046ad7132d159f1f40e7.txt b/_already_sent/a3ffc5f64551046ad7132d159f1f40e7.txt deleted file mode 100644 index 2e5d8665..00000000 --- a/_already_sent/a3ffc5f64551046ad7132d159f1f40e7.txt +++ /dev/null @@ -1 +0,0 @@ -https://goblackcat.com/feeling-exhausted/ \ No newline at end of file diff --git a/_already_sent/db41d26877002dfa9dba650122b8a298.txt b/_already_sent/db41d26877002dfa9dba650122b8a298.txt deleted file mode 100644 index 8b80b665..00000000 --- a/_already_sent/db41d26877002dfa9dba650122b8a298.txt +++ /dev/null @@ -1 +0,0 @@ -https://fenati.org.br/brasil-prepara-marco-regulatorio-para-data-centers-com-beneficios-fiscais-e-regras-sustentaveis/#datacenter \ No newline at end of file From b5adf7dbd076053bd738d2ef32d6200af3b14ba3 Mon Sep 17 00:00:00 2001 From: Santiago Lema Date: Wed, 21 May 2025 02:24:15 +0000 Subject: [PATCH 03/12] Handles token refrsh for oAUTH API --- add_to_fedilist.php | 48 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 43 insertions(+), 5 deletions(-) diff --git a/add_to_fedilist.php b/add_to_fedilist.php index 74a2b7d5..df246b0d 100644 --- a/add_to_fedilist.php +++ b/add_to_fedilist.php @@ -8,24 +8,62 @@ function isYouTubeLink($url) { } function addVideoToFediList($videoUrl) { - - $playlistId = @file_get_contents(__DIR__ .'/_credentials/fedilist_id.txt'); + + $credentialsDir = __DIR__ . '/_credentials'; + $tokenPath = "$credentialsDir/token.json"; + $playlistId = @file_get_contents("$credentialsDir/fedilist_id.txt"); + if (!$playlistId) { echo "ā€¼ļø Error: FediList ID not found. Make sure fedilist_id.txt exists.\n"; return false; } - $token = @json_decode(file_get_contents(__DIR__ .'/_credentials/token.json'), true); + // === Load client credentials === + $secrets = json_decode(file_get_contents("$credentialsDir/client_secret.json"), true); + $client = $secrets['installed'] ?? $secrets['web'] ?? null; + + if (!$client || !isset($client['client_id'], $client['client_secret'])) { + die("Error: Invalid client_secret.json format.\n"); + } + + $clientId = $client['client_id']; + $clientSecret = $client['client_secret']; + + // === Load token === + $token = @json_decode(file_get_contents($tokenPath), true); if (!$token || !isset($token['access_token'])) { echo "ā€¼ļø Error: token.json missing or invalid. Authenticate first.\n"; return false; } - // Extract video ID from URL + // === Refresh token if expired === + if (isset($token['expires_at']) && time() >= $token['expires_at']) { + echo "šŸ”„ Access token expired. Refreshing...\n"; + + $refreshResponse = curlPost('https://oauth2.googleapis.com/token', [ + 'client_id' => $clientId, + 'client_secret' => $clientSecret, + 'refresh_token' => $token['refresh_token'], + 'grant_type' => 'refresh_token' + ]); + + if (isset($refreshResponse['access_token'])) { + $token['access_token'] = $refreshResponse['access_token']; + $token['expires_in'] = $refreshResponse['expires_in']; + $token['expires_at'] = time() + $refreshResponse['expires_in']; + file_put_contents($tokenPath, json_encode($token)); + echo "āœ… Token refreshed.\n"; + } else { + echo "ā€¼ļø Failed to refresh token: " . ($refreshResponse['error'] ?? 'unknown') . "\n"; + return false; + } + } + + // === Extract video ID === if (!preg_match('/(?:v=|\/)([a-zA-Z0-9_-]{11})/', $videoUrl, $matches)) { echo "ā‰ļø Invalid YouTube URL: $videoUrl\n"; return false; - } + } $videoId = $matches[1]; // === Step 1: Check if video is already in playlist === From dee25226765467409f8c67a2beef5317f95f3b41 Mon Sep 17 00:00:00 2001 From: Santiago Lema Date: Wed, 21 May 2025 02:36:26 +0000 Subject: [PATCH 04/12] handles Google refresh token --- add_to_fedilist.php | 12 +++++++++++- update_google_token.php | 9 +++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/add_to_fedilist.php b/add_to_fedilist.php index df246b0d..d9910859 100644 --- a/add_to_fedilist.php +++ b/add_to_fedilist.php @@ -35,9 +35,17 @@ function addVideoToFediList($videoUrl) { echo "ā€¼ļø Error: token.json missing or invalid. Authenticate first.\n"; return false; } + + //print_r($token); // === Refresh token if expired === - if (isset($token['expires_at']) && time() >= $token['expires_at']) { + if (isset($token['expires_at'])) { + $secondsLeft = $token['expires_at'] - time(); + if ($secondsLeft > 0) { + $minutes = floor($secondsLeft / 60); + $seconds = $secondsLeft % 60; + echo "ā³ Token expires in $minutes minutes and $seconds seconds.\n"; + } else { echo "šŸ”„ Access token expired. Refreshing...\n"; $refreshResponse = curlPost('https://oauth2.googleapis.com/token', [ @@ -58,6 +66,8 @@ function addVideoToFediList($videoUrl) { return false; } } +} + // === Extract video ID === if (!preg_match('/(?:v=|\/)([a-zA-Z0-9_-]{11})/', $videoUrl, $matches)) { diff --git a/update_google_token.php b/update_google_token.php index f183070a..d6fe5412 100755 --- a/update_google_token.php +++ b/update_google_token.php @@ -109,12 +109,13 @@ while (true) { ]); if (isset($tokenResponse['access_token'])) { - $token = $tokenResponse; - echo "Saving token.json\n"; - file_put_contents(__DIR__ . '/_credentials/token.json', json_encode($token)); - break; + $tokenResponse['expires_at'] = time() + $tokenResponse['expires_in']; + file_put_contents(__DIR__ . '/_credentials/token.json', json_encode($tokenResponse)); + echo "āœ… Token saved with expiration.\n"; + break; } + if (isset($tokenResponse['error']) && $tokenResponse['error'] !== 'authorization_pending') { die("Auth error: " . $tokenResponse['error'] . "\n"); } From 920c8932a0ecf7dc7ce89f795e3ab495c08ae520 Mon Sep 17 00:00:00 2001 From: Santiago Lema Date: Wed, 21 May 2025 03:05:53 +0000 Subject: [PATCH 05/12] cleaner login process --- add_to_fedilist.php | 12 ++++++++++-- update_google_token.php | 42 ++++++++++++++++------------------------- utils.php | 21 +++++++++++++++++++++ 3 files changed, 47 insertions(+), 28 deletions(-) create mode 100644 utils.php diff --git a/add_to_fedilist.php b/add_to_fedilist.php index d9910859..a9aff48a 100644 --- a/add_to_fedilist.php +++ b/add_to_fedilist.php @@ -1,8 +1,11 @@ 0) { + if ($secondsLeft > $refreshSecondMargin) { $minutes = floor($secondsLeft / 60); $seconds = $secondsLeft % 60; echo "ā³ Token expires in $minutes minutes and $seconds seconds.\n"; } else { - echo "šŸ”„ Access token expired. Refreshing...\n"; + echo "šŸ”„ Access token expired or will expire in less than $refreshSecondMargin seconds. (Seconds Left: $secondsLeft). Refreshing...\n"; $refreshResponse = curlPost('https://oauth2.googleapis.com/token', [ 'client_id' => $clientId, diff --git a/update_google_token.php b/update_google_token.php index d6fe5412..82ed8075 100755 --- a/update_google_token.php +++ b/update_google_token.php @@ -1,6 +1,8 @@ #!/usr/bin/php $clientId, @@ -110,22 +99,23 @@ while (true) { if (isset($tokenResponse['access_token'])) { $tokenResponse['expires_at'] = time() + $tokenResponse['expires_in']; - file_put_contents(__DIR__ . '/_credentials/token.json', json_encode($tokenResponse)); - echo "āœ… Token saved with expiration.\n"; + $path = __DIR__ . '/_credentials/token.json'; + file_put_contents($path, json_encode($tokenResponse)); + echo "\nāœ… Token saved as $path.\n"; break; } if (isset($tokenResponse['error']) && $tokenResponse['error'] !== 'authorization_pending') { - die("Auth error: " . $tokenResponse['error'] . "\n"); + die("\nAuth error: " . $tokenResponse['error'] . "\n"); } if (time() - $startTime > $deviceData['expires_in']) { - die("Authorization timed out.\n"); + die("\nAuthorization timed out.\n"); } } // === Step 3: Create FediList Playlist === -createPlaylist($token['access_token']); +createPlaylist($tokenResponse['access_token']); ?> diff --git a/utils.php b/utils.php new file mode 100644 index 00000000..39efbd04 --- /dev/null +++ b/utils.php @@ -0,0 +1,21 @@ + From 0d3b7594f78b09f9424b487fe1258f7b8808599c Mon Sep 17 00:00:00 2001 From: Santiago Lema Date: Thu, 21 Aug 2025 01:35:21 +0000 Subject: [PATCH 06/12] update --- fedi_slurp.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fedi_slurp.php b/fedi_slurp.php index a473b6b8..04426e4d 100755 --- a/fedi_slurp.php +++ b/fedi_slurp.php @@ -62,9 +62,10 @@ foreach ($bookmarks as $status) { } } - +if ($links) echo "Valid URLS:".count($links)."\n"; - +else + echo "NO links founds \n"; print_r($links); From 8b6d35779ee6bc198270f9c57c969d81a6f59e95 Mon Sep 17 00:00:00 2001 From: Santiago Lema Date: Thu, 21 Aug 2025 02:40:31 +0000 Subject: [PATCH 07/12] supports multiple accounts, removed harcoded account on top, added _credentials_example --- _credentials_example/.gitkeep | 0 _credentials_example/client_secret.json | 2 + _credentials_example/fedi_accounts.txt | 2 + _credentials_example/fedilist_id.txt | 1 + _credentials_example/readeck_account.txt | 1 + _credentials_example/token.json | 1 + fedi_slurp.php | 87 +++++++++++++++++++++--- 7 files changed, 84 insertions(+), 10 deletions(-) create mode 100644 _credentials_example/.gitkeep create mode 100644 _credentials_example/client_secret.json create mode 100644 _credentials_example/fedi_accounts.txt create mode 100644 _credentials_example/fedilist_id.txt create mode 100644 _credentials_example/readeck_account.txt create mode 100644 _credentials_example/token.json diff --git a/_credentials_example/.gitkeep b/_credentials_example/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/_credentials_example/client_secret.json b/_credentials_example/client_secret.json new file mode 100644 index 00000000..bfda3599 --- /dev/null +++ b/_credentials_example/client_secret.json @@ -0,0 +1,2 @@ +{"installed":{"client_id":"XYZ.apps.googleusercontent.com","project_id":"yourproject_with_youtube_access","auth_uri":"https://accounts.google.com/o/oauth2/auth","token_uri":"https://oauth2.googleapis.com/token","auth_provider_x509_cert_url":"https://www.googleapis.com/oauth2/v1/certs","client_secret":"XYZXYZ"}} + diff --git a/_credentials_example/fedi_accounts.txt b/_credentials_example/fedi_accounts.txt new file mode 100644 index 00000000..d6f6fa63 --- /dev/null +++ b/_credentials_example/fedi_accounts.txt @@ -0,0 +1,2 @@ +your.instance.org|XXXYYY +another.instance.com.br|XXXYYY diff --git a/_credentials_example/fedilist_id.txt b/_credentials_example/fedilist_id.txt new file mode 100644 index 00000000..4859b3e9 --- /dev/null +++ b/_credentials_example/fedilist_id.txt @@ -0,0 +1 @@ +XXXXXXXYYYYYYYYY diff --git a/_credentials_example/readeck_account.txt b/_credentials_example/readeck_account.txt new file mode 100644 index 00000000..d0c57fe2 --- /dev/null +++ b/_credentials_example/readeck_account.txt @@ -0,0 +1 @@ +your.readeck.fr|XXXXXYYYY diff --git a/_credentials_example/token.json b/_credentials_example/token.json new file mode 100644 index 00000000..c2c648ed --- /dev/null +++ b/_credentials_example/token.json @@ -0,0 +1 @@ +{"access_token":"XXXYYY","expires_in":3599,"refresh_token":"1\/\/AAABBBCCC","scope":"https:\/\/www.googleapis.com\/auth\/youtube","token_type":"Bearer","expires_at":1755745256} diff --git a/fedi_slurp.php b/fedi_slurp.php index 04426e4d..1f8437e2 100755 --- a/fedi_slurp.php +++ b/fedi_slurp.php @@ -1,31 +1,63 @@ #!/usr/bin/php true, + CURLOPT_USERAGENT => "FediSlurperScript/1.0 (https://code.lema.org/santiago/fedi_slurp)", + CURLOPT_HTTPHEADER => [ "Authorization: Bearer $MASTODON_TOKEN", "Accept: application/json" @@ -40,6 +72,8 @@ if (!is_array($bookmarks)) { echo "Found bookmarks:".count($bookmarks)."\n"; +#print_r($bookmarks); + //----------------------------- // FIND VALID URLs in posts //----------------------------- @@ -62,12 +96,17 @@ foreach ($bookmarks as $status) { } } -if ($links) +if (isset($links)) +{ echo "Valid URLS:".count($links)."\n"; -else - echo "NO links founds \n"; print_r($links); +} +else +{ + echo "NO links founds. Kthxbye \n"; + die(0); +} //----------------------------- // SEND LINKS TO READECK @@ -92,7 +131,6 @@ if (!is_dir($alreadySentDir)) { mkdir($alreadySentDir, 0755, true); // recursive mkdir } -require("add_to_fedilist.php"); foreach ($links as $link) { @@ -177,3 +215,32 @@ if (isYouTubeLink($link)) { } curl_close($ch); + +} // end accounts loop + + +function loadAccounts(string $filepath): array { + $accounts = []; + + if (!file_exists($filepath)) { + return $accounts; // empty if file not found + } + + $lines = file($filepath, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES); + + foreach ($lines as $line) { + $line = trim($line); + if ($line === '') continue; + + [$host, $token] = explode('|', $line, 2); + $accounts[] = [ + 'host' => $host, + 'token' => $token + ]; + } + + return $accounts; +} + + + From 2b25421f5cee622a8d27125a4eafa8f6956f614c Mon Sep 17 00:00:00 2001 From: santiago Date: Thu, 21 Aug 2025 02:47:31 +0000 Subject: [PATCH 08/12] Update README.md --- README.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b92f7413..d0f12bd4 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Fedi Link Fetcher -A pair of PHP scripts that extract links from your Mastodon / snac bookmarks and add them to: +A pair of PHP scripts that extract links from your Mastodon / gotosocial / snac bookmarks and add them to: - Read Later in Readeck (simple API token) - Watch Later in YouTube Playlist (Google oAUTH API) @@ -12,12 +12,17 @@ In Web UI: - Settings, API tokens, Create API Token - Check Bookmarks Read + Write -Save it as: +Save it in: ``` -/_credentials/token.json +/_credentials/readeck_account.txt ``` +Just add one line like: +readeck.instance.com|YOUR_TOKEN -## 2. Getting a Youtube API token + +## + +## 3. Getting a Youtube API token (it's a tad more complicated...) - Go to the Google Cloud Console: - Project > APIs & Services > Credentials @@ -28,6 +33,7 @@ Save it as: Save it as: ``` /_credentials/client_secret.json +/_credentials/token.json ``` ## 3. Add your Google account as a test user From c2df91467915d24a42c615666c091d8eee5160a3 Mon Sep 17 00:00:00 2001 From: santiago Date: Thu, 21 Aug 2025 02:51:25 +0000 Subject: [PATCH 09/12] Updated readme to explain how to get tokens --- README.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d0f12bd4..dbac06f7 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,20 @@ Just add one line like: readeck.instance.com|YOUR_TOKEN -## +## 2. Getting mastodon / snac / gotosocial tokens + +You can either user the respective web UI or just use the [Token Generator here](https://takahashim.github.io/mastodon-access-token/) , just set the URL, login, and get the token back + +Save one account per line in: +``` +/_credentials/fedi_accounts.txt +``` +Just add one line per user like this (you can have several times the same instance as the token determines the user): +mastodon.social|YOUR_TOKEN +mastodon.social|YOUR_TOKEN +my.instance.org|YOUR_TOKEN + +The script will loop on each account but always save on the same readeck / youtube accounts. ## 3. Getting a Youtube API token (it's a tad more complicated...) From e53b4866b87df6a73cd9df7ec20834a5e041e2e3 Mon Sep 17 00:00:00 2001 From: santiago Date: Thu, 21 Aug 2025 02:52:47 +0000 Subject: [PATCH 10/12] Update README.md --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index dbac06f7..1182b395 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,9 @@ Save it in: /_credentials/readeck_account.txt ``` Just add one line like: +``` readeck.instance.com|YOUR_TOKEN +``` ## 2. Getting mastodon / snac / gotosocial tokens @@ -29,9 +31,11 @@ Save one account per line in: /_credentials/fedi_accounts.txt ``` Just add one line per user like this (you can have several times the same instance as the token determines the user): +``` mastodon.social|YOUR_TOKEN mastodon.social|YOUR_TOKEN my.instance.org|YOUR_TOKEN +``` The script will loop on each account but always save on the same readeck / youtube accounts. From 1aaae0c4e31d7daae5f478f8c0cf14dc38c6c145 Mon Sep 17 00:00:00 2001 From: Santiago Lema Date: Thu, 21 Aug 2025 03:02:34 +0000 Subject: [PATCH 11/12] clean up php with php-cs-fixer --- .php-cs-fixer.cache | 1 + fedi_slurp.php | 335 ++++++++++++++++++++++---------------------- 2 files changed, 167 insertions(+), 169 deletions(-) create mode 100644 .php-cs-fixer.cache diff --git a/.php-cs-fixer.cache b/.php-cs-fixer.cache new file mode 100644 index 00000000..bc404ecb --- /dev/null +++ b/.php-cs-fixer.cache @@ -0,0 +1 @@ +{"php":"8.2.29","version":"3.86.0:v3.86.0#4a952bd19dc97879b0620f495552ef09b55f7d36","indent":" ","lineEnding":"\n","rules":{"binary_operator_spaces":{"default":"at_least_single_space"},"blank_line_after_opening_tag":true,"blank_line_between_import_groups":true,"blank_lines_before_namespace":true,"braces_position":{"allow_single_line_empty_anonymous_classes":true},"class_definition":{"inline_constructor_arguments":false,"space_before_parenthesis":true},"compact_nullable_type_declaration":true,"declare_equal_normalize":true,"lowercase_cast":true,"lowercase_static_reference":true,"new_with_parentheses":true,"no_blank_lines_after_class_opening":true,"no_extra_blank_lines":{"tokens":["use"]},"no_leading_import_slash":true,"no_whitespace_in_blank_line":true,"ordered_class_elements":{"order":["use_trait"]},"ordered_imports":{"imports_order":["class","function","const"],"sort_algorithm":"none"},"return_type_declaration":true,"short_scalar_cast":true,"single_import_per_statement":{"group_to_single_imports":false},"single_space_around_construct":{"constructs_followed_by_a_single_space":["abstract","as","case","catch","class","const_import","do","else","elseif","final","finally","for","foreach","function","function_import","if","insteadof","interface","namespace","new","private","protected","public","static","switch","trait","try","use","use_lambda","while"],"constructs_preceded_by_a_single_space":["as","else","elseif","use_lambda"]},"single_trait_insert_per_statement":true,"ternary_operator_spaces":true,"unary_operator_spaces":{"only_dec_inc":true},"visibility_required":true,"blank_line_after_namespace":true,"constant_case":true,"control_structure_braces":true,"control_structure_continuation_position":true,"elseif":true,"function_declaration":true,"indentation_type":true,"line_ending":true,"lowercase_keywords":true,"method_argument_space":{"attribute_placement":"ignore","on_multiline":"ensure_fully_multiline"},"no_break_comment":true,"no_closing_tag":true,"no_multiple_statements_per_line":true,"no_space_around_double_colon":true,"no_spaces_after_function_name":true,"no_trailing_whitespace":true,"no_trailing_whitespace_in_comment":true,"single_blank_line_at_eof":true,"single_class_element_per_statement":{"elements":["property"]},"single_line_after_imports":true,"spaces_inside_parentheses":true,"statement_indentation":true,"switch_case_semicolon_to_colon":true,"switch_case_space":true,"encoding":true,"full_opening_tag":true},"hashes":{"fedi_slurp.php":"2cbe2b549c703ecb4a3be4f2e15694f9"}} \ No newline at end of file diff --git a/fedi_slurp.php b/fedi_slurp.php index 1f8437e2..0c33318d 100755 --- a/fedi_slurp.php +++ b/fedi_slurp.php @@ -13,7 +13,7 @@ $MINIMUM_TEXT_SIZE = 500; // article with less characters of content will be ig $fediAccounts = loadAccounts(__DIR__ . '/_credentials/fedi_accounts.txt'); $readeckAccount = loadAccounts(__DIR__ . '/_credentials/readeck_account.txt'); -// _credentials/readeck_account.txt +// _credentials/readeck_account.txt // should have only one line with host|token // ex: gone.lema.org|XXXXYYYXXXYYY @@ -27,199 +27,197 @@ echo "Fedi Accounts to loop: ".count($fediAccounts)."\n"; // _credentials/fedi_accountst.txt -// each line like with host|token +// each line like with host|token // ex: gotosocial.lema.org|XXXXYYYXXXYYY foreach ($fediAccounts as $acc) { $MASTODON_HOST = $acc['host']; $MASTODON_TOKEN = $acc['token']; - echo ""; - echo ""; - echo "--------------------------------\n"; - echo "Host: $MASTODON_HOST\n"; - echo "Token: $MASTODON_TOKEN\n"; - echo "--------------------------------\n"; - echo ""; + echo ""; + echo ""; + echo "--------------------------------\n"; + echo "Host: $MASTODON_HOST\n"; + echo "Token: $MASTODON_TOKEN\n"; + echo "--------------------------------\n"; + echo ""; -//----------------------------- -// FETCH MASTODON BOOKMARKS -//----------------------------- -echo "# Fetching mastodon / gotosocial / snac bookmarks...\n"; -date_default_timezone_set('America/Sao_Paulo'); -echo date('Y-m-d H:i:s')."\n"; + //----------------------------- + // FETCH MASTODON BOOKMARKS + //----------------------------- + echo "# Fetching mastodon / gotosocial / snac bookmarks...\n"; + date_default_timezone_set('America/Sao_Paulo'); + echo date('Y-m-d H:i:s')."\n"; -$ch = curl_init("https://$MASTODON_HOST/api/v1/bookmarks"); + $ch = curl_init("https://$MASTODON_HOST/api/v1/bookmarks"); -#GotoSocial will reply with error "I am a teapot" if no user agent is sent... -curl_setopt_array($ch, [ - CURLOPT_RETURNTRANSFER => true, - CURLOPT_USERAGENT => "FediSlurperScript/1.0 (https://code.lema.org/santiago/fedi_slurp)", + #GotoSocial will reply with error "I am a teapot" if no user agent is sent... + curl_setopt_array($ch, [ + CURLOPT_RETURNTRANSFER => true, + CURLOPT_USERAGENT => "FediSlurperScript/1.0 (https://code.lema.org/santiago/fedi_slurp)", - CURLOPT_HTTPHEADER => [ - "Authorization: Bearer $MASTODON_TOKEN", - "Accept: application/json" - ] -]); - -$bookmarksJson = curl_exec($ch); -$bookmarks = json_decode($bookmarksJson, true); -if (!is_array($bookmarks)) { - die("āŒ Failed to parse Mastodon bookmarks.\n"); -} - -echo "Found bookmarks:".count($bookmarks)."\n"; - -#print_r($bookmarks); - -//----------------------------- -// FIND VALID URLs in posts -//----------------------------- - -foreach ($bookmarks as $status) { - if (!isset($status['content'])) { - continue; - } - $content = strip_tags($status['content']); - preg_match_all('/https?:\/\/[^\s"<]+/', $content, $matches); - if (!empty($matches[0])) { - - $oneLink = $matches[0][0]; - if (filter_var($oneLink, FILTER_VALIDATE_URL)) { - $links[] = $oneLink; - } else { - // This happens for example if URL has an emoji at the end - echo "INVALID URL: $oneLink\n"; - } - } -} - -if (isset($links)) -{ -echo "Valid URLS:".count($links)."\n"; -print_r($links); -} -else -{ - echo "NO links founds. Kthxbye \n"; - die(0); - -} - -//----------------------------- -// SEND LINKS TO READECK -//----------------------------- - -$apiUrl = "https://$READECK_HOST/api/bookmarks"; - -$ch = curl_init(); -curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); -curl_setopt($ch, CURLOPT_POST, true); -curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0'); - -$headers = [ - "Authorization: Bearer $READECK_TOKEN", - 'Accept: application/json', - 'Content-Type: application/json' -]; - -$alreadySentDir = __DIR__ . "/_already_sent"; - -if (!is_dir($alreadySentDir)) { - mkdir($alreadySentDir, 0755, true); // recursive mkdir -} - - - -foreach ($links as $link) { - -if (isYouTubeLink($link)) { - addVideoToFediList($link); - continue; -} - - - - // READECK will accept several times the same URL ! - // Make sure we don't send it several times by keeping an archive here - $hash = md5($link); - $filePath = __DIR__ . "/_already_sent/{$hash}.txt"; - - if (file_exists($filePath)) { - echo "ā„¹ļø Already sent: $link\n"; - continue; - } - - - $options = [ - 'http' => [ - 'method' => 'GET', - 'header' => "User-Agent: Mozilla/5.0\r\n" - ] -]; - - // First check if page has content - //$ch = curl_init($link);; - curl_setopt($ch, CURLOPT_URL, $link); - curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - $content = curl_exec($ch); - - if ($content === false) { - echo "āŒ Failed to fetch $link\n"; - continue; - } - $plainText = strip_tags($content); - - if (strlen($plainText) < $MINIMUM_TEXT_SIZE) { - echo "āš ļø Skipping $link\ncontent too small (".strlen($plainText)." chars < $MINIMUM_TEXT_SIZE )\n"; - continue; - } - - echo "🟢 Will add to Readeck $link\nLength: " . strlen($plainText)."\n"; - - //not passing title here, since we don't have it - $payload = json_encode([ - "labels" => ["automasto"], - "url" => $link + CURLOPT_HTTPHEADER => [ + "Authorization: Bearer $MASTODON_TOKEN", + "Accept: application/json" + ] ]); - curl_setopt($ch, CURLOPT_URL, $apiUrl); - curl_setopt($ch, CURLOPT_POSTFIELDS, $payload); - curl_setopt($ch, CURLOPT_HTTPHEADER, $headers); + $bookmarksJson = curl_exec($ch); + $bookmarks = json_decode($bookmarksJson, true); + if (!is_array($bookmarks)) { + die("āŒ Failed to parse Mastodon bookmarks.\n"); + } - $response = curl_exec($ch); - $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE); + echo "Found bookmarks:".count($bookmarks)."\n"; + #print_r($bookmarks); - if (curl_errno($ch)) { - echo "āŒ Error adding $link: " . curl_error($ch) . "\n"; - } else { + //----------------------------- + // FIND VALID URLs in posts + //----------------------------- - // Store already sent file only if connection worked - file_put_contents($filePath, $link); + foreach ($bookmarks as $status) { + if (!isset($status['content'])) { + continue; + } + $content = strip_tags($status['content']); + preg_match_all('/https?:\/\/[^\s"<]+/', $content, $matches); + if (!empty($matches[0])) { - $json = json_decode($response, true); - if (json_last_error() === JSON_ERROR_NONE) { - if ($httpCode >= 200 && $httpCode < 300) { - echo "āœ… [$httpCode] Successfully added: $link\n"; + $oneLink = $matches[0][0]; + if (filter_var($oneLink, FILTER_VALIDATE_URL)) { + $links[] = $oneLink; } else { - echo "āš ļø Server returned status $httpCode for $link\n"; + // This happens for example if URL has an emoji at the end + echo "INVALID URL: $oneLink\n"; } - } else { - echo "āš ļø Response is not valid JSON for $link: $response\n"; } } -} + if (isset($links)) { + echo "Valid URLS:".count($links)."\n"; + print_r($links); + } else { + echo "NO links founds. Kthxbye \n"; + die(0); -curl_close($ch); + } + + //----------------------------- + // SEND LINKS TO READECK + //----------------------------- + + $apiUrl = "https://$READECK_HOST/api/bookmarks"; + + $ch = curl_init(); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + curl_setopt($ch, CURLOPT_POST, true); + curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0'); + + $headers = [ + "Authorization: Bearer $READECK_TOKEN", + 'Accept: application/json', + 'Content-Type: application/json' + ]; + + $alreadySentDir = __DIR__ . "/_already_sent"; + + if (!is_dir($alreadySentDir)) { + mkdir($alreadySentDir, 0755, true); // recursive mkdir + } + + + + foreach ($links as $link) { + + if (isYouTubeLink($link)) { + addVideoToFediList($link); + continue; + } + + + + // READECK will accept several times the same URL ! + // Make sure we don't send it several times by keeping an archive here + $hash = md5($link); + $filePath = __DIR__ . "/_already_sent/{$hash}.txt"; + + if (file_exists($filePath)) { + echo "ā„¹ļø Already sent: $link\n"; + continue; + } + + + $options = [ + 'http' => [ + 'method' => 'GET', + 'header' => "User-Agent: Mozilla/5.0\r\n" + ] +]; + + // First check if page has content + //$ch = curl_init($link);; + curl_setopt($ch, CURLOPT_URL, $link); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + $content = curl_exec($ch); + + if ($content === false) { + echo "āŒ Failed to fetch $link\n"; + continue; + } + $plainText = strip_tags($content); + + if (strlen($plainText) < $MINIMUM_TEXT_SIZE) { + echo "āš ļø Skipping $link\ncontent too small (".strlen($plainText)." chars < $MINIMUM_TEXT_SIZE )\n"; + continue; + } + + echo "🟢 Will add to Readeck $link\nLength: " . strlen($plainText)."\n"; + + //not passing title here, since we don't have it + $payload = json_encode([ + "labels" => ["automasto"], + "url" => $link + ]); + + curl_setopt($ch, CURLOPT_URL, $apiUrl); + curl_setopt($ch, CURLOPT_POSTFIELDS, $payload); + curl_setopt($ch, CURLOPT_HTTPHEADER, $headers); + + $response = curl_exec($ch); + $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE); + + + if (curl_errno($ch)) { + echo "āŒ Error adding $link: " . curl_error($ch) . "\n"; + } else { + + // Store already sent file only if connection worked + file_put_contents($filePath, $link); + + $json = json_decode($response, true); + if (json_last_error() === JSON_ERROR_NONE) { + if ($httpCode >= 200 && $httpCode < 300) { + echo "āœ… [$httpCode] Successfully added: $link\n"; + } else { + echo "āš ļø Server returned status $httpCode for $link\n"; + } + } else { + echo "āš ļø Response is not valid JSON for $link: $response\n"; + } + } + + } + + curl_close($ch); } // end accounts loop -function loadAccounts(string $filepath): array { +function loadAccounts(string $filepath): array +{ $accounts = []; if (!file_exists($filepath)) { @@ -230,7 +228,9 @@ function loadAccounts(string $filepath): array { foreach ($lines as $line) { $line = trim($line); - if ($line === '') continue; + if ($line === '') { + continue; + } [$host, $token] = explode('|', $line, 2); $accounts[] = [ @@ -241,6 +241,3 @@ function loadAccounts(string $filepath): array { return $accounts; } - - - From faf3e9f929c39aa5e86f3e1166e072d2ed76cb7b Mon Sep 17 00:00:00 2001 From: Santiago Lema Date: Thu, 21 Aug 2025 03:05:31 +0000 Subject: [PATCH 12/12] ran php-cs-fixer on all files --- .php-cs-fixer.cache | 1 - update_google_token.php | 33 ++++++++++++++++----------------- 2 files changed, 16 insertions(+), 18 deletions(-) delete mode 100644 .php-cs-fixer.cache diff --git a/.php-cs-fixer.cache b/.php-cs-fixer.cache deleted file mode 100644 index bc404ecb..00000000 --- a/.php-cs-fixer.cache +++ /dev/null @@ -1 +0,0 @@ -{"php":"8.2.29","version":"3.86.0:v3.86.0#4a952bd19dc97879b0620f495552ef09b55f7d36","indent":" ","lineEnding":"\n","rules":{"binary_operator_spaces":{"default":"at_least_single_space"},"blank_line_after_opening_tag":true,"blank_line_between_import_groups":true,"blank_lines_before_namespace":true,"braces_position":{"allow_single_line_empty_anonymous_classes":true},"class_definition":{"inline_constructor_arguments":false,"space_before_parenthesis":true},"compact_nullable_type_declaration":true,"declare_equal_normalize":true,"lowercase_cast":true,"lowercase_static_reference":true,"new_with_parentheses":true,"no_blank_lines_after_class_opening":true,"no_extra_blank_lines":{"tokens":["use"]},"no_leading_import_slash":true,"no_whitespace_in_blank_line":true,"ordered_class_elements":{"order":["use_trait"]},"ordered_imports":{"imports_order":["class","function","const"],"sort_algorithm":"none"},"return_type_declaration":true,"short_scalar_cast":true,"single_import_per_statement":{"group_to_single_imports":false},"single_space_around_construct":{"constructs_followed_by_a_single_space":["abstract","as","case","catch","class","const_import","do","else","elseif","final","finally","for","foreach","function","function_import","if","insteadof","interface","namespace","new","private","protected","public","static","switch","trait","try","use","use_lambda","while"],"constructs_preceded_by_a_single_space":["as","else","elseif","use_lambda"]},"single_trait_insert_per_statement":true,"ternary_operator_spaces":true,"unary_operator_spaces":{"only_dec_inc":true},"visibility_required":true,"blank_line_after_namespace":true,"constant_case":true,"control_structure_braces":true,"control_structure_continuation_position":true,"elseif":true,"function_declaration":true,"indentation_type":true,"line_ending":true,"lowercase_keywords":true,"method_argument_space":{"attribute_placement":"ignore","on_multiline":"ensure_fully_multiline"},"no_break_comment":true,"no_closing_tag":true,"no_multiple_statements_per_line":true,"no_space_around_double_colon":true,"no_spaces_after_function_name":true,"no_trailing_whitespace":true,"no_trailing_whitespace_in_comment":true,"single_blank_line_at_eof":true,"single_class_element_per_statement":{"elements":["property"]},"single_line_after_imports":true,"spaces_inside_parentheses":true,"statement_indentation":true,"switch_case_semicolon_to_colon":true,"switch_case_space":true,"encoding":true,"full_opening_tag":true},"hashes":{"fedi_slurp.php":"2cbe2b549c703ecb4a3be4f2e15694f9"}} \ No newline at end of file diff --git a/update_google_token.php b/update_google_token.php index 82ed8075..71469ce5 100755 --- a/update_google_token.php +++ b/update_google_token.php @@ -15,13 +15,14 @@ $clientId = $client['client_id']; $clientSecret = $client['client_secret']; // === Create playlist helper === -function createPlaylist($accessToken) { +function createPlaylist($accessToken) +{ $path = __DIR__ . '/_credentials/fedilist_id.txt'; - if (file_exists($path)) { - echo "āœ… FediList Playlist ID already exists at :$path\n"; - return; + if (file_exists($path)) { + echo "āœ… FediList Playlist ID already exists at :$path\n"; + return; } $data = [ @@ -53,12 +54,12 @@ function createPlaylist($accessToken) { $result = json_decode($response, true); -if (isset($result['id'])) { - echo "FediList created!\nPlaylist ID: " . $result['id'] . "\n"; - file_put_contents($path, $result['id']); -} else { - echo "Failed to create playlist:\n$response\n"; -} + if (isset($result['id'])) { + echo "FediList created!\nPlaylist ID: " . $result['id'] . "\n"; + file_put_contents($path, $result['id']); + } else { + echo "Failed to create playlist:\n$response\n"; + } } @@ -98,11 +99,11 @@ while (true) { ]); if (isset($tokenResponse['access_token'])) { - $tokenResponse['expires_at'] = time() + $tokenResponse['expires_in']; - $path = __DIR__ . '/_credentials/token.json'; - file_put_contents($path, json_encode($tokenResponse)); - echo "\nāœ… Token saved as $path.\n"; - break; + $tokenResponse['expires_at'] = time() + $tokenResponse['expires_in']; + $path = __DIR__ . '/_credentials/token.json'; + file_put_contents($path, json_encode($tokenResponse)); + echo "\nāœ… Token saved as $path.\n"; + break; } @@ -117,5 +118,3 @@ while (true) { // === Step 3: Create FediList Playlist === createPlaylist($tokenResponse['access_token']); - -?>