Compare commits
No commits in common. "1b81034873fecb3427406d5de95f6c3b77233413" and "99e1fe28189ca9089c9620c5551437d2ee13e9e2" have entirely different histories.
1b81034873
...
99e1fe2818
7 changed files with 9 additions and 1714 deletions
13
.gitignore
vendored
13
.gitignore
vendored
|
|
@ -1,8 +1,9 @@
|
||||||
/already_sent/*
|
# ---> ThinkPHP
|
||||||
!/already_sent/.gitkeep
|
# gitignore template for ThinkPHP v3.2.3
|
||||||
/_credentials/*
|
# website: http://www.thinkphp.cn/
|
||||||
!/_credentials/.gitkeep
|
|
||||||
/_vendor/*
|
|
||||||
/vendor/
|
|
||||||
|
|
||||||
|
# Logs and Cache files
|
||||||
|
/Application/Runtime/
|
||||||
|
|
||||||
|
# Common configure file
|
||||||
|
/Application/Common/Conf/config.php
|
||||||
|
|
|
||||||
35
README.md
35
README.md
|
|
@ -1,34 +1,3 @@
|
||||||
|
# fedi_slurp
|
||||||
|
|
||||||
# Fedi Link Fetcher
|
A bunch of scripts to fetch your Mastodon / snac bookmarks (using Mastodon API) and save them in your Readeck account for reading later or Youtube account for watching later.
|
||||||
|
|
||||||
A pair of PHP scripts that extract links from your Mastodon / snac bookmarks and add them to:
|
|
||||||
|
|
||||||
- YouTube Playlist (Google oAUTH API)
|
|
||||||
- Readeck (simple API token)
|
|
||||||
|
|
||||||
## TODO instructions readeck API
|
|
||||||
|
|
||||||
how to get tokeb in web ui
|
|
||||||
|
|
||||||
## TODO isntructions Youtube PlayList
|
|
||||||
|
|
||||||
|
|
||||||
1. Go to the Google Cloud Console:
|
|
||||||
• Project > APIs & Services > Credentials
|
|
||||||
|
|
||||||
create oauth client ID
|
|
||||||
for TV and limited INPUT
|
|
||||||
• Download the OAuth 2.0 Client ID JSON file
|
|
||||||
|
|
||||||
save as /home/snac/youtube_list/client_secret.json
|
|
||||||
|
|
||||||
Option 1: Add your Google account as a test user
|
|
||||||
1. Go to Google Cloud Console > OAuth consent screen
|
|
||||||
~> Audience
|
|
||||||
2. Scroll to “Test users”
|
|
||||||
3. Add your Google email (e.g. jacques.lema@gmail.com)
|
|
||||||
|
|
||||||
|
|
||||||
composer require google/apiclient:^2.0
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -1,93 +0,0 @@
|
||||||
<?php
|
|
||||||
|
|
||||||
// === Example use ===
|
|
||||||
#addVideoToFediList('https://www.youtube.com/watch?v=dQw4w9WgXcQ');
|
|
||||||
|
|
||||||
function isYouTubeLink($url) {
|
|
||||||
return preg_match('#^(https?://)?(www\.)?(youtube\.com/watch\?v=|youtu\.be/)[a-zA-Z0-9_-]{11}#', $url);
|
|
||||||
}
|
|
||||||
|
|
||||||
function addVideoToFediList($videoUrl) {
|
|
||||||
|
|
||||||
$playlistId = @file_get_contents(__DIR__ .'/_credentials/fedilist_id.txt');
|
|
||||||
if (!$playlistId) {
|
|
||||||
echo "‼️ Error: FediList ID not found. Make sure fedilist_id.txt exists.\n";
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
$token = @json_decode(file_get_contents(__DIR__ .'/_credentials/token.json'), true);
|
|
||||||
if (!$token || !isset($token['access_token'])) {
|
|
||||||
echo "‼️ Error: token.json missing or invalid. Authenticate first.\n";
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Extract video ID from URL
|
|
||||||
if (!preg_match('/(?:v=|\/)([a-zA-Z0-9_-]{11})/', $videoUrl, $matches)) {
|
|
||||||
echo "⁉️ Invalid YouTube URL: $videoUrl\n";
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
$videoId = $matches[1];
|
|
||||||
|
|
||||||
// === Step 1: Check if video is already in playlist ===
|
|
||||||
$checkUrl = "https://www.googleapis.com/youtube/v3/playlistItems?part=snippet&playlistId={$playlistId}&maxResults=50";
|
|
||||||
$found = false;
|
|
||||||
|
|
||||||
do {
|
|
||||||
$ch = curl_init($checkUrl);
|
|
||||||
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
|
|
||||||
curl_setopt($ch, CURLOPT_HTTPHEADER, [
|
|
||||||
'Authorization: Bearer ' . $token['access_token'],
|
|
||||||
'User-Agent: curl/7.64.1'
|
|
||||||
]);
|
|
||||||
$response = curl_exec($ch);
|
|
||||||
curl_close($ch);
|
|
||||||
|
|
||||||
$data = json_decode($response, true);
|
|
||||||
|
|
||||||
foreach ($data['items'] ?? [] as $item) {
|
|
||||||
if (($item['snippet']['resourceId']['videoId'] ?? '') === $videoId) {
|
|
||||||
echo "ℹ️ Video already in FediList: $videoId\n";
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
$checkUrl = $data['nextPageToken'] ?? false
|
|
||||||
? $checkUrl . '&pageToken=' . $data['nextPageToken']
|
|
||||||
: false;
|
|
||||||
|
|
||||||
} while ($checkUrl);
|
|
||||||
|
|
||||||
// === Step 2: Add the video ===
|
|
||||||
$postData = [
|
|
||||||
'snippet' => [
|
|
||||||
'playlistId' => $playlistId,
|
|
||||||
'resourceId' => [
|
|
||||||
'kind' => 'youtube#video',
|
|
||||||
'videoId' => $videoId
|
|
||||||
]
|
|
||||||
]
|
|
||||||
];
|
|
||||||
|
|
||||||
$ch = curl_init('https://www.googleapis.com/youtube/v3/playlistItems?part=snippet');
|
|
||||||
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
|
|
||||||
curl_setopt($ch, CURLOPT_HTTPHEADER, [
|
|
||||||
'Authorization: Bearer ' . $token['access_token'],
|
|
||||||
'Content-Type: application/json',
|
|
||||||
'User-Agent: curl/7.64.1'
|
|
||||||
]);
|
|
||||||
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($postData));
|
|
||||||
$response = curl_exec($ch);
|
|
||||||
curl_close($ch);
|
|
||||||
|
|
||||||
$result = json_decode($response, true);
|
|
||||||
|
|
||||||
if (isset($result['id'])) {
|
|
||||||
echo "✅ Added video to FediList: $videoId\n";
|
|
||||||
return true;
|
|
||||||
} else {
|
|
||||||
echo "‼️ Failed to add video:\n$response\n";
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
?>
|
|
||||||
|
|
@ -1,5 +0,0 @@
|
||||||
{
|
|
||||||
"require": {
|
|
||||||
"google/apiclient": "^2.0"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
1281
composer.lock
generated
1281
composer.lock
generated
File diff suppressed because it is too large
Load diff
|
|
@ -1,118 +0,0 @@
|
||||||
#!/usr/bin/php
|
|
||||||
<?php
|
|
||||||
|
|
||||||
// === Load client credentials ===
|
|
||||||
$secrets = json_decode(file_get_contents(__DIR__ . '/_credentials/client_secret.json'), true);
|
|
||||||
$client = $secrets['installed'] ?? $secrets['web'] ?? null;
|
|
||||||
|
|
||||||
if (!$client || !isset($client['client_id'], $client['client_secret'])) {
|
|
||||||
die("Error: Invalid client_secret.json format.\n");
|
|
||||||
}
|
|
||||||
|
|
||||||
$clientId = $client['client_id'];
|
|
||||||
$clientSecret = $client['client_secret'];
|
|
||||||
|
|
||||||
// === cURL helper function ===
|
|
||||||
function curlPost($url, $data, $headers = []) {
|
|
||||||
$ch = curl_init($url);
|
|
||||||
curl_setopt($ch, CURLOPT_POST, true);
|
|
||||||
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
|
|
||||||
curl_setopt($ch, CURLOPT_HTTPHEADER, array_merge([
|
|
||||||
'Content-Type: application/x-www-form-urlencoded',
|
|
||||||
'User-Agent: curl/7.64.1'
|
|
||||||
], $headers));
|
|
||||||
curl_setopt($ch, CURLOPT_POSTFIELDS, http_build_query($data));
|
|
||||||
$response = curl_exec($ch);
|
|
||||||
if (curl_errno($ch)) {
|
|
||||||
die("cURL error: " . curl_error($ch) . "\n");
|
|
||||||
}
|
|
||||||
curl_close($ch);
|
|
||||||
return json_decode($response, true);
|
|
||||||
}
|
|
||||||
|
|
||||||
// === Create playlist helper ===
|
|
||||||
function createPlaylist($accessToken) {
|
|
||||||
$data = [
|
|
||||||
'snippet' => [
|
|
||||||
'title' => 'FediList',
|
|
||||||
'description' => 'A custom playlist for federated videos'
|
|
||||||
],
|
|
||||||
'status' => [
|
|
||||||
'privacyStatus' => 'private'
|
|
||||||
]
|
|
||||||
];
|
|
||||||
|
|
||||||
$ch = curl_init('https://www.googleapis.com/youtube/v3/playlists?part=snippet,status');
|
|
||||||
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
|
|
||||||
curl_setopt($ch, CURLOPT_HTTPHEADER, [
|
|
||||||
"Authorization: Bearer $accessToken",
|
|
||||||
"Content-Type: application/json",
|
|
||||||
"User-Agent: curl/7.64.1"
|
|
||||||
]);
|
|
||||||
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($data));
|
|
||||||
$response = curl_exec($ch);
|
|
||||||
if (curl_errno($ch)) {
|
|
||||||
die("cURL error: " . curl_error($ch) . "\n");
|
|
||||||
}
|
|
||||||
curl_close($ch);
|
|
||||||
|
|
||||||
$result = json_decode($response, true);
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
$result = json_decode($response, true);
|
|
||||||
if (isset($result['id'])) {
|
|
||||||
echo "FediList created!\nPlaylist ID: " . $result['id'] . "\n";
|
|
||||||
file_put_contents(__DIR__ . '/_credentials/fedilist_id.txt', $result['id']);
|
|
||||||
} else {
|
|
||||||
echo "Failed to create playlist:\n$response\n";
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
// === Step 1: Get device code ===
|
|
||||||
$deviceData = curlPost('https://oauth2.googleapis.com/device/code', [
|
|
||||||
'client_id' => $clientId,
|
|
||||||
'scope' => 'https://www.googleapis.com/auth/youtube'
|
|
||||||
]);
|
|
||||||
|
|
||||||
if (!isset($deviceData['user_code'])) {
|
|
||||||
die("Failed to get device code.\n");
|
|
||||||
}
|
|
||||||
|
|
||||||
echo "==== DEVICE AUTHORIZATION ====\n";
|
|
||||||
echo "Visit: " . $deviceData['verification_url'] . "\n";
|
|
||||||
echo "Enter code: " . $deviceData['user_code'] . "\n\n";
|
|
||||||
|
|
||||||
// === Step 2: Poll for token ===
|
|
||||||
$token = null;
|
|
||||||
$startTime = time();
|
|
||||||
while (true) {
|
|
||||||
sleep($deviceData['interval']);
|
|
||||||
|
|
||||||
$tokenResponse = curlPost('https://oauth2.googleapis.com/token', [
|
|
||||||
'client_id' => $clientId,
|
|
||||||
'client_secret' => $clientSecret,
|
|
||||||
'device_code' => $deviceData['device_code'],
|
|
||||||
'grant_type' => 'urn:ietf:params:oauth:grant-type:device_code'
|
|
||||||
]);
|
|
||||||
|
|
||||||
if (isset($tokenResponse['access_token'])) {
|
|
||||||
$token = $tokenResponse;
|
|
||||||
file_put_contents(__DIR__ . '/_credentials/token.json', json_encode($token));
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (isset($tokenResponse['error']) && $tokenResponse['error'] !== 'authorization_pending') {
|
|
||||||
die("Auth error: " . $tokenResponse['error'] . "\n");
|
|
||||||
}
|
|
||||||
|
|
||||||
if (time() - $startTime > $deviceData['expires_in']) {
|
|
||||||
die("Authorization timed out.\n");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// === Step 3: Create FediList Playlist ===
|
|
||||||
createPlaylist($token['access_token']);
|
|
||||||
|
|
||||||
?>
|
|
||||||
178
fedi_slurp.php
178
fedi_slurp.php
|
|
@ -1,178 +0,0 @@
|
||||||
#!/usr/bin/php
|
|
||||||
<?php
|
|
||||||
|
|
||||||
//-----------------------------
|
|
||||||
// CREDENTIALS
|
|
||||||
//-----------------------------
|
|
||||||
|
|
||||||
$MASTODON_TOKEN = '8beea62e32b336e5d934d06a21b0b996';
|
|
||||||
$MASTODON_HOST = 'go.lema.org';
|
|
||||||
|
|
||||||
$READECK_TOKEN = 'LDJb4YbGKe6Fp8cSygpuw5LjmwkgGTAbFbP77TQtYwe1hFZ4';
|
|
||||||
$READECK_HOST = 'read.lema.org';
|
|
||||||
|
|
||||||
$MINIMUM_TEXT_SIZE = 500; // article with less characters of content will be ignored
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
//-----------------------------
|
|
||||||
// FETCH MASTODON BOOKMARKS
|
|
||||||
//-----------------------------
|
|
||||||
echo "# Fetching mastodon / snac bookmarks...\n";
|
|
||||||
date_default_timezone_set('America/Sao_Paulo');
|
|
||||||
echo date('Y-m-d H:i:s')."\n";
|
|
||||||
|
|
||||||
$ch = curl_init("https://$MASTODON_HOST/api/v1/bookmarks");
|
|
||||||
curl_setopt_array($ch, [
|
|
||||||
CURLOPT_RETURNTRANSFER => true,
|
|
||||||
CURLOPT_HTTPHEADER => [
|
|
||||||
"Authorization: Bearer $MASTODON_TOKEN",
|
|
||||||
"Accept: application/json"
|
|
||||||
]
|
|
||||||
]);
|
|
||||||
|
|
||||||
$bookmarksJson = curl_exec($ch);
|
|
||||||
$bookmarks = json_decode($bookmarksJson, true);
|
|
||||||
if (!is_array($bookmarks)) {
|
|
||||||
die("❌ Failed to parse Mastodon bookmarks.\n");
|
|
||||||
}
|
|
||||||
|
|
||||||
echo "Found bookmarks:".count($bookmarks)."\n";
|
|
||||||
|
|
||||||
//-----------------------------
|
|
||||||
// FIND VALID URLs in posts
|
|
||||||
//-----------------------------
|
|
||||||
|
|
||||||
foreach ($bookmarks as $status) {
|
|
||||||
if (!isset($status['content'])) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
$content = strip_tags($status['content']);
|
|
||||||
preg_match_all('/https?:\/\/[^\s"<]+/', $content, $matches);
|
|
||||||
if (!empty($matches[0])) {
|
|
||||||
|
|
||||||
$oneLink = $matches[0][0];
|
|
||||||
if (filter_var($oneLink, FILTER_VALIDATE_URL)) {
|
|
||||||
$links[] = $oneLink;
|
|
||||||
} else {
|
|
||||||
// This happens for example if URL has an emoji at the end
|
|
||||||
echo "INVALID URL: $oneLink\n";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
echo "Valid URLS:".count($links)."\n";
|
|
||||||
|
|
||||||
print_r($links);
|
|
||||||
|
|
||||||
|
|
||||||
//-----------------------------
|
|
||||||
// SEND LINKS TO READECK
|
|
||||||
//-----------------------------
|
|
||||||
|
|
||||||
$apiUrl = "https://$READECK_HOST/api/bookmarks";
|
|
||||||
|
|
||||||
$ch = curl_init();
|
|
||||||
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
|
|
||||||
curl_setopt($ch, CURLOPT_POST, true);
|
|
||||||
curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0');
|
|
||||||
|
|
||||||
$headers = [
|
|
||||||
"Authorization: Bearer $READECK_TOKEN",
|
|
||||||
'Accept: application/json',
|
|
||||||
'Content-Type: application/json'
|
|
||||||
];
|
|
||||||
|
|
||||||
$alreadySentDir = __DIR__ . "/already_sent";
|
|
||||||
|
|
||||||
if (!is_dir($alreadySentDir)) {
|
|
||||||
mkdir($alreadySentDir, 0755, true); // recursive mkdir
|
|
||||||
}
|
|
||||||
|
|
||||||
require("add_to_fedilist.php");
|
|
||||||
|
|
||||||
|
|
||||||
foreach ($links as $link) {
|
|
||||||
|
|
||||||
if (isYouTubeLink($link)) {
|
|
||||||
addVideoToFediList($link);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
// READECK will accept several times the same URL !
|
|
||||||
// Make sure we don't send it several times by keeping an archive here
|
|
||||||
$hash = md5($link);
|
|
||||||
$filePath = __DIR__ . "/already_sent/{$hash}.txt";
|
|
||||||
|
|
||||||
if (file_exists($filePath)) {
|
|
||||||
echo "ℹ️ Already sent: $link\n";
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
$options = [
|
|
||||||
'http' => [
|
|
||||||
'method' => 'GET',
|
|
||||||
'header' => "User-Agent: Mozilla/5.0\r\n"
|
|
||||||
]
|
|
||||||
];
|
|
||||||
|
|
||||||
// First check if page has content
|
|
||||||
//$ch = curl_init($link);;
|
|
||||||
curl_setopt($ch, CURLOPT_URL, $link);
|
|
||||||
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
|
|
||||||
$content = curl_exec($ch);
|
|
||||||
|
|
||||||
if ($content === false) {
|
|
||||||
echo "❌ Failed to fetch $link\n";
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
$plainText = strip_tags($content);
|
|
||||||
|
|
||||||
if (strlen($plainText) < $MINIMUM_TEXT_SIZE) {
|
|
||||||
echo "⚠️ Skipping $link\ncontent too small (".strlen($plainText)." chars < $MINIMUM_TEXT_SIZE )\n";
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
echo "🟢 Will add to Readeck $link\nLength: " . strlen($plainText)."\n";
|
|
||||||
|
|
||||||
//not passing title here, since we don't have it
|
|
||||||
$payload = json_encode([
|
|
||||||
"labels" => ["automasto"],
|
|
||||||
"url" => $link
|
|
||||||
]);
|
|
||||||
|
|
||||||
curl_setopt($ch, CURLOPT_URL, $apiUrl);
|
|
||||||
curl_setopt($ch, CURLOPT_POSTFIELDS, $payload);
|
|
||||||
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
|
|
||||||
|
|
||||||
$response = curl_exec($ch);
|
|
||||||
$httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
|
|
||||||
|
|
||||||
|
|
||||||
if (curl_errno($ch)) {
|
|
||||||
echo "❌ Error adding $link: " . curl_error($ch) . "\n";
|
|
||||||
} else {
|
|
||||||
|
|
||||||
// Store already sent file only if connection worked
|
|
||||||
file_put_contents($filePath, $link);
|
|
||||||
|
|
||||||
$json = json_decode($response, true);
|
|
||||||
if (json_last_error() === JSON_ERROR_NONE) {
|
|
||||||
if ($httpCode >= 200 && $httpCode < 300) {
|
|
||||||
echo "✅ [$httpCode] Successfully added: $link\n";
|
|
||||||
} else {
|
|
||||||
echo "⚠️ Server returned status $httpCode for $link\n";
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
echo "⚠️ Response is not valid JSON for $link: $response\n";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
curl_close($ch);
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue