Stop tracking vendor folder

.gitignore

@@ -1,9 +1,8 @@
-# ---> ThinkPHP
-# gitignore template for ThinkPHP v3.2.3
-# website: http://www.thinkphp.cn/
+/already_sent/*
+!/already_sent/.gitkeep
+/_credentials/*
+!/_credentials/.gitkeep
+/_vendor/*
+/vendor/
 
-# Logs and Cache files
-/Application/Runtime/
 
-# Common configure file
-/Application/Common/Conf/config.php

README.md

@@ -1,3 +1,34 @@
-# fedi_slurp
 
-A bunch of scripts to fetch your Mastodon / snac bookmarks (using Mastodon API) and save them in your Readeck account for reading later or Youtube account for watching later.
+# Fedi Link Fetcher
+
+A pair of PHP scripts that extract links from your Mastodon / snac bookmarks and add them to:
+
+- YouTube Playlist (Google oAUTH API)
+- Readeck (simple API token) 
+
+## TODO instructions readeck API 
+
+how to get tokeb in web ui
+
+## TODO isntructions Youtube PlayList
+
+
+	1.	Go to the Google Cloud Console:
+	•	Project > APIs & Services > Credentials
+
+create oauth client ID 
+for TV and limited INPUT
+	•	Download the OAuth 2.0 Client ID JSON file
+
+save as /home/snac/youtube_list/client_secret.json
+
+Option 1: Add your Google account as a test user
+	1.	Go to Google Cloud Console > OAuth consent screen
+~> Audience
+	2.	Scroll to “Test users”
+	3.	Add your Google email (e.g. jacques.lema@gmail.com)
+
+
+composer require google/apiclient:^2.0
+
+

add_to_fedilist.php

@@ -0,0 +1,93 @@
+<?php
+
+// === Example use ===
+#addVideoToFediList('https://www.youtube.com/watch?v=dQw4w9WgXcQ');
+
+function isYouTubeLink($url) {
+    return preg_match('#^(https?://)?(www\.)?(youtube\.com/watch\?v=|youtu\.be/)[a-zA-Z0-9_-]{11}#', $url);
+}
+
+function addVideoToFediList($videoUrl) {
+		
+    $playlistId = @file_get_contents(__DIR__ .'/_credentials/fedilist_id.txt');
+    if (!$playlistId) {
+        echo "‼️ Error: FediList ID not found. Make sure fedilist_id.txt exists.\n";
+        return false;
+    }
+
+    $token = @json_decode(file_get_contents(__DIR__ .'/_credentials/token.json'), true);
+    if (!$token || !isset($token['access_token'])) {
+        echo "‼️ Error: token.json missing or invalid. Authenticate first.\n";
+        return false;
+    }
+
+    // Extract video ID from URL
+    if (!preg_match('/(?:v=|\/)([a-zA-Z0-9_-]{11})/', $videoUrl, $matches)) {
+        echo "⁉️ Invalid YouTube URL: $videoUrl\n";
+        return false;
+    }
+    $videoId = $matches[1];
+
+    // === Step 1: Check if video is already in playlist ===
+    $checkUrl = "https://www.googleapis.com/youtube/v3/playlistItems?part=snippet&playlistId={$playlistId}&maxResults=50";
+    $found = false;
+
+    do {
+        $ch = curl_init($checkUrl);
+        curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
+        curl_setopt($ch, CURLOPT_HTTPHEADER, [
+            'Authorization: Bearer ' . $token['access_token'],
+            'User-Agent: curl/7.64.1'
+        ]);
+        $response = curl_exec($ch);
+        curl_close($ch);
+
+        $data = json_decode($response, true);
+
+        foreach ($data['items'] ?? [] as $item) {
+            if (($item['snippet']['resourceId']['videoId'] ?? '') === $videoId) {
+                echo "ℹ️ Video already in FediList: $videoId\n";
+                return false;
+            }
+        }
+
+        $checkUrl = $data['nextPageToken'] ?? false
+            ? $checkUrl . '&pageToken=' . $data['nextPageToken']
+            : false;
+
+    } while ($checkUrl);
+
+    // === Step 2: Add the video ===
+    $postData = [
+        'snippet' => [
+            'playlistId' => $playlistId,
+            'resourceId' => [
+                'kind' => 'youtube#video',
+                'videoId' => $videoId
+            ]
+        ]
+    ];
+
+    $ch = curl_init('https://www.googleapis.com/youtube/v3/playlistItems?part=snippet');
+    curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
+    curl_setopt($ch, CURLOPT_HTTPHEADER, [
+        'Authorization: Bearer ' . $token['access_token'],
+        'Content-Type: application/json',
+        'User-Agent: curl/7.64.1'
+    ]);
+    curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($postData));
+    $response = curl_exec($ch);
+    curl_close($ch);
+
+    $result = json_decode($response, true);
+
+    if (isset($result['id'])) {
+        echo "✅ Added video to FediList: $videoId\n";
+        return true;
+    } else {
+        echo "‼️ Failed to add video:\n$response\n";
+        return false;
+    }
+}
+
+?>

composer.json

@@ -0,0 +1,5 @@
+{
+    "require": {
+        "google/apiclient": "^2.0"
+    }
+}

create_fedilist.php

@@ -0,0 +1,118 @@
+#!/usr/bin/php
+<?php
+
+// === Load client credentials ===
+$secrets = json_decode(file_get_contents(__DIR__ . '/_credentials/client_secret.json'), true);
+$client = $secrets['installed'] ?? $secrets['web'] ?? null;
+
+if (!$client || !isset($client['client_id'], $client['client_secret'])) {
+    die("Error: Invalid client_secret.json format.\n");
+}
+
+$clientId = $client['client_id'];
+$clientSecret = $client['client_secret'];
+
+// === cURL helper function ===
+function curlPost($url, $data, $headers = []) {
+    $ch = curl_init($url);
+    curl_setopt($ch, CURLOPT_POST, true);
+    curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
+    curl_setopt($ch, CURLOPT_HTTPHEADER, array_merge([
+        'Content-Type: application/x-www-form-urlencoded',
+        'User-Agent: curl/7.64.1'
+    ], $headers));
+    curl_setopt($ch, CURLOPT_POSTFIELDS, http_build_query($data));
+    $response = curl_exec($ch);
+    if (curl_errno($ch)) {
+        die("cURL error: " . curl_error($ch) . "\n");
+    }
+    curl_close($ch);
+    return json_decode($response, true);
+}
+
+// === Create playlist helper ===
+function createPlaylist($accessToken) {
+    $data = [
+        'snippet' => [
+            'title' => 'FediList',
+            'description' => 'A custom playlist for federated videos'
+        ],
+        'status' => [
+            'privacyStatus' => 'private'
+        ]
+    ];
+
+    $ch = curl_init('https://www.googleapis.com/youtube/v3/playlists?part=snippet,status');
+    curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
+    curl_setopt($ch, CURLOPT_HTTPHEADER, [
+        "Authorization: Bearer $accessToken",
+        "Content-Type: application/json",
+        "User-Agent: curl/7.64.1"
+    ]);
+    curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($data));
+    $response = curl_exec($ch);
+    if (curl_errno($ch)) {
+        die("cURL error: " . curl_error($ch) . "\n");
+    }
+    curl_close($ch);
+
+    $result = json_decode($response, true);
+
+
+
+    $result = json_decode($response, true);
+if (isset($result['id'])) {
+    echo "FediList created!\nPlaylist ID: " . $result['id'] . "\n";
+    file_put_contents(__DIR__ . '/_credentials/fedilist_id.txt', $result['id']);
+} else {
+    echo "Failed to create playlist:\n$response\n";
+}
+
+}
+
+// === Step 1: Get device code ===
+$deviceData = curlPost('https://oauth2.googleapis.com/device/code', [
+    'client_id' => $clientId,
+    'scope' => 'https://www.googleapis.com/auth/youtube'
+]);
+
+if (!isset($deviceData['user_code'])) {
+    die("Failed to get device code.\n");
+}
+
+echo "==== DEVICE AUTHORIZATION ====\n";
+echo "Visit:      " . $deviceData['verification_url'] . "\n";
+echo "Enter code: " . $deviceData['user_code'] . "\n\n";
+
+// === Step 2: Poll for token ===
+$token = null;
+$startTime = time();
+while (true) {
+    sleep($deviceData['interval']);
+
+    $tokenResponse = curlPost('https://oauth2.googleapis.com/token', [
+        'client_id' => $clientId,
+        'client_secret' => $clientSecret,
+        'device_code' => $deviceData['device_code'],
+        'grant_type' => 'urn:ietf:params:oauth:grant-type:device_code'
+    ]);
+
+    if (isset($tokenResponse['access_token'])) {
+        $token = $tokenResponse;
+        file_put_contents(__DIR__ . '/_credentials/token.json', json_encode($token));
+        break;
+    }
+
+    if (isset($tokenResponse['error']) && $tokenResponse['error'] !== 'authorization_pending') {
+        die("Auth error: " . $tokenResponse['error'] . "\n");
+    }
+
+    if (time() - $startTime > $deviceData['expires_in']) {
+        die("Authorization timed out.\n");
+    }
+}
+
+// === Step 3: Create FediList Playlist ===
+createPlaylist($token['access_token']);
+
+?>

fedi_slurp.php

@@ -0,0 +1,178 @@
+#!/usr/bin/php
+<?php
+
+//-----------------------------
+// CREDENTIALS
+//-----------------------------
+
+$MASTODON_TOKEN    = '8beea62e32b336e5d934d06a21b0b996';
+$MASTODON_HOST     = 'go.lema.org';
+ 
+$READECK_TOKEN     = 'LDJb4YbGKe6Fp8cSygpuw5LjmwkgGTAbFbP77TQtYwe1hFZ4';
+$READECK_HOST      = 'read.lema.org';
+
+$MINIMUM_TEXT_SIZE = 500;  // article with less characters of content will be ignored
+
+
+
+
+//-----------------------------
+// FETCH MASTODON BOOKMARKS
+//-----------------------------
+echo "# Fetching mastodon / snac bookmarks...\n";
+date_default_timezone_set('America/Sao_Paulo');
+echo date('Y-m-d H:i:s')."\n";
+
+$ch = curl_init("https://$MASTODON_HOST/api/v1/bookmarks");
+curl_setopt_array($ch, [
+    CURLOPT_RETURNTRANSFER => true,
+    CURLOPT_HTTPHEADER => [
+        "Authorization: Bearer $MASTODON_TOKEN",
+        "Accept: application/json"
+    ]
+]);
+
+$bookmarksJson = curl_exec($ch);
+$bookmarks = json_decode($bookmarksJson, true);
+if (!is_array($bookmarks)) {
+    die("❌ Failed to parse Mastodon bookmarks.\n");
+}
+
+echo "Found bookmarks:".count($bookmarks)."\n";
+
+//-----------------------------
+// FIND VALID URLs in posts
+//-----------------------------
+
+foreach ($bookmarks as $status) {
+    if (!isset($status['content'])) {
+        continue;
+    }
+    $content = strip_tags($status['content']);
+    preg_match_all('/https?:\/\/[^\s"<]+/', $content, $matches);
+    if (!empty($matches[0])) {
+
+        $oneLink = $matches[0][0];
+        if (filter_var($oneLink, FILTER_VALIDATE_URL)) {
+            $links[] = $oneLink;
+        } else {
+            // This happens for example if URL has an emoji at the end
+            echo "INVALID URL: $oneLink\n";
+        }
+    }
+}
+
+
+echo "Valid URLS:".count($links)."\n";
+
+print_r($links);
+
+
+//-----------------------------
+// SEND LINKS TO READECK
+//-----------------------------
+
+$apiUrl = "https://$READECK_HOST/api/bookmarks";
+
+$ch = curl_init();
+curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
+curl_setopt($ch, CURLOPT_POST, true);
+curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0');
+
+$headers = [
+    "Authorization: Bearer $READECK_TOKEN",
+    'Accept: application/json',
+    'Content-Type: application/json'
+];
+
+$alreadySentDir = __DIR__ . "/already_sent";
+
+if (!is_dir($alreadySentDir)) {
+    mkdir($alreadySentDir, 0755, true); // recursive mkdir
+}
+
+require("add_to_fedilist.php");
+
+
+foreach ($links as $link) {
+
+if (isYouTubeLink($link)) {
+    addVideoToFediList($link);
+    continue;
+}
+
+
+
+    // READECK will accept several times the same URL !
+    // Make sure we don't send it several times by keeping an archive here
+    $hash = md5($link);
+    $filePath = __DIR__ . "/already_sent/{$hash}.txt";
+
+    if (file_exists($filePath)) {
+        echo "ℹ️  Already sent: $link\n";
+        continue;
+    }
+
+
+    $options = [
+    'http' => [
+        'method' => 'GET',
+        'header' => "User-Agent: Mozilla/5.0\r\n"
+    ]
+];
+
+    // First check if page has content
+    //$ch = curl_init($link);;
+    curl_setopt($ch, CURLOPT_URL, $link);
+    curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
+    $content = curl_exec($ch);
+
+    if ($content === false) {
+        echo "❌ Failed to fetch $link\n";
+        continue;
+    }
+    $plainText = strip_tags($content);
+
+    if (strlen($plainText) < $MINIMUM_TEXT_SIZE) {
+        echo "⚠️ Skipping $link\ncontent too small (".strlen($plainText)." chars < $MINIMUM_TEXT_SIZE )\n";
+        continue;
+    }
+
+    echo "🟢 Will add to Readeck $link\nLength: " . strlen($plainText)."\n";
+
+    //not passing title here, since we don't have it
+    $payload = json_encode([
+        "labels" => ["automasto"],
+        "url" => $link
+    ]);
+
+    curl_setopt($ch, CURLOPT_URL, $apiUrl);
+    curl_setopt($ch, CURLOPT_POSTFIELDS, $payload);
+    curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
+
+    $response = curl_exec($ch);
+    $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
+
+
+    if (curl_errno($ch)) {
+        echo "❌ Error adding $link: " . curl_error($ch) . "\n";
+    } else {
+
+        // Store already sent file only if connection worked
+        file_put_contents($filePath, $link);
+
+        $json = json_decode($response, true);
+        if (json_last_error() === JSON_ERROR_NONE) {
+            if ($httpCode >= 200 && $httpCode < 300) {
+                echo "✅ [$httpCode] Successfully added: $link\n";
+            } else {
+                echo "⚠️ Server returned status $httpCode for $link\n";
+            }
+        } else {
+            echo "⚠️ Response is not valid JSON for $link: $response\n";
+        }
+    }
+
+}
+
+curl_close($ch);