add tool。

Tools/UrlCheck.cpp

@@ -0,0 +1,121 @@
+#include "Base/Database.h"
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/ip/tcp.hpp>
+#include <boost/asio/ssl/context.hpp>
+#include <boost/beast/core/flat_buffer.hpp>
+#include <boost/beast/core/tcp_stream.hpp>
+#include <boost/beast/http/message.hpp>
+#include <boost/beast/http/read.hpp>
+#include <boost/beast/http/string_body.hpp>
+#include <boost/beast/http/write.hpp>
+#include <boost/beast/ssl/ssl_stream.hpp>
+#include <boost/beast/version.hpp>
+#include <boost/program_options/options_description.hpp>
+#include <boost/program_options/parsers.hpp>
+#include <boost/program_options/variables_map.hpp>
+#include <filesystem>
+#include <iostream>
+
+bool isUrlValid(boost::asio::io_context &ioContext, const std::string &host, const std::string &port, const std::string &target);
+
+int main(int argc, char const *argv[]) {
+    boost::program_options::options_description description("Allowed options");
+    // clang-format off
+    description.add_options()
+            ("help,h", "produce help message.")
+            ("database,d", boost::program_options::value<std::string>(),"set database path")
+            ("delete-invalid", boost::program_options::value<bool>()->default_value(false),"delete invalid url");
+    // clang-format on
+    boost::program_options::variables_map values;
+    boost::program_options::store(boost::program_options::parse_command_line(argc, argv, description), values);
+    boost::program_options::notify(values);
+
+    std::string path;
+
+    if (values.count("help")) {
+        std::cout << description << std::endl;
+        std::exit(0);
+    } else if (values.count("database")) {
+        path = values.at("database").as<std::string>();
+    }
+
+    if (path.empty()) {
+        std::cerr << "please specify the database path." << std::endl;
+        std::cout << description << std::endl;
+        return 1;
+    } else if (!std::filesystem::exists(path)) {
+        std::cerr << "database file " << path << " not existed." << std::endl;
+        return 2;
+    }
+
+    Older::Database database;
+    if (!database.open(path)) {
+        return 3;
+    }
+
+    boost::asio::io_context ioContext;
+    auto items = database.visitRecords();
+    for (auto &item : items) {
+        bool valid = isUrlValid(ioContext, "amass.fun", "443", item.url);
+        std::cout << item.url << std::endl;
+        std::cout << "valid: " << valid << std::endl;
+        if (!valid && values.at("delete-invalid").as<bool>()) {
+            std::cout << "delete: " << database.removeVisitRecord(item.id) << std::endl;
+        }
+        std::cout << "----------" << std::endl;
+    }
+
+    return 0;
+}
+
+bool isUrlValid(boost::asio::io_context &ioContext, const std::string &host, const std::string &port, const std::string &target) {
+    using namespace boost;
+    using namespace boost::asio;
+    using namespace boost::asio::ip;
+    using namespace boost::beast;
+    try {
+        // 1. 创建SSL上下文
+        ssl::context ssl_ctx(ssl::context::tlsv12_client);
+        ssl_ctx.set_default_verify_paths();
+        ssl_ctx.set_verify_mode(ssl::verify_peer);
+
+        // 2. 创建TCP解析器和SSL流
+        tcp::resolver resolver(ioContext);
+        beast::ssl_stream<beast::tcp_stream> stream(ioContext, ssl_ctx);
+
+        // 3. 设置SNI主机名（重要！）
+        if (!SSL_set_tlsext_host_name(stream.native_handle(), host.c_str())) {
+            throw boost::system::system_error(::ERR_get_error(), boost::asio ::error::get_ssl_category());
+        }
+
+        // 4. 解析主机名并建立连接
+        auto const results = resolver.resolve(host, port);
+        beast::get_lowest_layer(stream).connect(results);
+        stream.handshake(ssl::stream_base::client);
+
+        // 5. 构造并发送HEAD请求
+        http::request<http::string_body> req{http::verb::get, target, 11};
+        req.set(http::field::host, host);
+        req.set(http::field::user_agent, BOOST_BEAST_VERSION_STRING);
+        http::write(stream, req);
+
+        // 6. 读取响应
+        beast::flat_buffer buffer;
+        http::response<http::string_body> res;
+        http::read(stream, buffer, res);
+        // 7. 检查HTTP状态码（2xx/3xx视为可访问）
+        const unsigned status = res.result_int();
+        const bool accessible = (status >= 200 && status < 400);
+
+        // 8. 优雅关闭连接
+        beast::error_code ec;
+        stream.shutdown(ec);
+        if (ec == net::error::eof || ec == boost::asio::ssl::error::stream_truncated) {
+            ec = {};
+        }
+        return accessible;
+    } catch (const std::exception &e) {
+        std::cerr << "Error: " << e.what() << std::endl;
+        return false;
+    }
+}