Table of Contents

1. 创建滑动解锁页面
   1. 准备三张图片
   2. 使用NEXTJS构建页面
2. 创建识别服务
3. 使用puppeteer 工具在网页上滑动验证
   1. NOTES:

创建滑动解锁页面

准备三张图片

• 背景图片

• 缺口图片

• 滑块图片
  

使用NEXTJS构建页面

  const SliderCaptcha = ({ onVerify }: SliderCaptchaProps) => {
  const [isDragging, setIsDragging] = useState(false)
  const [stPos,setStPos] = useState(0)
  const [position, setPosition] = useState(0)
  const containerRef = useRef(null)
  const handleStart = (e) => {
    console.log(e)
    setIsDragging(true)
  }

  const handleMove = (e) => {
    if (isDragging && containerRef.current) {
      let clientX = e.clientX || (e.touches && e.touches[0].clientX)
      console.log('move', clientX)
      setPosition(clientX)
    }
  }

  const handleEnd = () => {
    setIsDragging(false)
    const correctX = 120
    if (Math.abs(position - correctX) 
      

        
        
      
      

        
      
    
  )
 }

export { SliderCaptcha }

创建识别服务

• 利用yolov8,label-studio 训练模型，识别背景缺口位置

• 搭建识别服务python + flask

  !/usr/bin/env python3

  """Simple Flask server for captcha detect."""

  Importing flask module in the project is mandatory

  An object of Flask class is our WSGI application.

  from flask import Flask
  from flask import request, jsonify
  import cv2
  import numpy as np
  import base64
  import onnxruntime as ort
  import json

  Load the ONNX model

  ONNX_MODEL_PATH = "runs/train/exp/weights/best.onnx"
  session = ort.InferenceSession(ONNX_MODEL_PATH)

  Get model input details

  input_name = session.get_inputs()[0].name
  input_shape = session.get_inputs()[0].shape # e.g., [1, 3, 320, 320]
  img_size = input_shape[2] # Assuming square input (e.g., 320)

  Class names (from your data.yaml)

  class_names = ["target"] # Replace with your classes

  Preprocess the image (unchanged)

  def preprocess_image(img, img_size):
  """Preprocess image function."""
  orig_shape = img.shape[:2] # (height, width)
  ratio = min(img_size / orig_shape[0], img_size / orig_shape[1])
  new_shape = (int(orig_shape[1] * ratio), int(orig_shape[0] * ratio))
  img_resized = cv2.resize(img, new_shape, interpolation=cv2.INTER_LINEAR)
  img_padded = np.zeros((img_size, img_size, 3), dtype=np.uint8)
  top = (img_size - new_shape[1]) // 2
  left = (img_size - new_shape[0]) // 2
  img_padded[top : top + new_shape[1], left : left + new_shape[0]] = img_resized
  img_input = img_padded.astype(np.float32) / 255.0
  img_input = img_input.transpose(2, 0, 1)
  img_input = np.expand_dims(img_input, axis=0)
  return img_input, orig_shape, ratio, (top, left)

  def postprocess_output(
  output, orig_shape, ratio, padding, conf_thres=0.25, iou_thres=0.45
  ):
  print("Raw output shape:", output.shape)

  # Transpose to [n_detections, 5]
  predictions = output[0].T  # [2100, 5]
  print("Predictions shape:", predictions.shape)
  
  # Extract boxes and confidence
  boxes = predictions[:, :4]  # [x_center, y_center, width, height]
  conf = predictions[:, 4]  # Confidence scores
  
  # Filter by confidence
  mask = conf > conf_thres
  boxes = boxes[mask]
  scores = conf[mask]  # Use conf as scores since no class probs
  
  print("Filtered detections:", boxes.shape[0])
  
  # Handle case with no detections
  if boxes.shape[0] == 0:
      print("No detections above confidence threshold.")
      return []
  
  # For single-class or no class scores, assume class_id = 0 (or adjust if multi-class elsewhere)
  class_ids = np.zeros(len(scores), dtype=int)  # Default to class 0
  
  # NMS (Non-Max Suppression)
  indices = cv2.dnn.NMSBoxes(boxes.tolist(), scores.tolist(), conf_thres, iou_thres)
  
  # Adjust boxes to original image coordinates
  detections = []
  top, left = padding
  for i in indices:
      box = boxes[i]
      score = scores[i]
      class_id = class_ids[i]
  
      x_center, y_center, w, h = box
      x = (x_center - w / 2 - left) / ratio
      y = (y_center - h / 2 - top) / ratio
      w /= ratio
      h /= ratio
  
      x = max(0, min(x, orig_shape[1]))
      y = max(0, min(y, orig_shape[0]))
      w = min(w, orig_shape[1] - x)
      h = min(h, orig_shape[0] - y)
  
      detections.append((class_names[class_id], score, (x, y, w, h)))
  
  return detections
  

  Main inference function (unchanged)

  def detect_image(img):
  """Detect image function."""
  img_input, orig_shape, ratio, padding = preprocess_image(img, img_size)
  outputs = session.run(None, {input_name: img_input})
  detections = postprocess_output(outputs[0], orig_shape, ratio, padding)
  result = {"rects": [], "img": ""}
  # img = cv2.imread(image_path)
  for class_name, score, (x, y, w, h) in detections:
  x, y, w, h = int(x), int(y), int(w), int(h)
  cv2.rectangle(img, (x, y), (x + w, y + h), (0, 255, 0), 2)
  label = f"{class_name}: {score:.2f}"
  cv2.putText(
  img, label, (x, y - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2
  )
  result["rects"].append((x, y, w, h))
  _, buffer = cv2.imencode(".jpeg", img)
  result["img"] = base64.b64encode(buffer).decode("utf-8")
  # cv2.imwrite("./detect1.jpeg", img)
  # print(f"Saved detection result to {'./detect.jpeg'}")
  return jsonify(result)

  app = Flask(name)

  @app.route("/")
  def hello_world():
  """Hello_World function."""
  print("Hello World")
  return "Hello World"

  @app.route("/detect", methods=["POST"])
  def detect_captcha():
  """Detect captcha function."""
  print("detect_captcha")
  base64_string = request.get_json()
  # base64_string.decode("utf-8").split("base64,")[1].strip("')")
  img_bytes = base64.b64decode(base64_string["data"])

  # Convert bytes to a NumPy array
  img_array = np.frombuffer(img_bytes, np.uint8)
  
  # Decode the image using cv2.imdecode
  image = cv2.imdecode(img_array, cv2.IMREAD_COLOR)
  return detect_image(image)
  

  if name == "main":
  app.run(debug=True, port=5001)

  

  广告

使用puppeteer 工具在网页上滑动验证

      async sloveCaptcha(page:Page,tryTimes:number){
        if(tryTimes {
            return el
                .getAttribute("src")
                ?.replace("data:image/jpg;base64,", "");
        });
        if (!cpc_image) return;
        const puzz = await axios
            .post("http://localhost:5001/detect", { data: cpc_image })
            .then((res) => {
                const result = res.data as { rects: any; img: string };
                console.log(result["rects"]);
                fspromises.writeFile(
                    "./cache/detected.jpeg",
                    result.img,
                    "base64",
                );
                return result.rects[0] || [];
            })
            .catch((err) => {
                console.error(err);
                return [];
            });
        if (puzz.length === 0) return;
        console.log(`puzzz ${JSON.stringify(puzz)}`);
        //page.mouse.move(x, y)
        const destCx = cpcBox.x+puzz[0]+puzz[2]/2;
        const moveHandler = await page.$("img.move-img");
        if (!moveHandler) {
            console.log("move img not found!");
            return;
        }
        const moveBox= await moveHandler.boundingBox();
        if (!moveBox) {
            console.log("bounding box error");
            return;
        }
        let xPos = moveBox.x + moveBox.width / 2;
        let yPos = moveBox.y + moveBox.height / 2;
        console.log(`entered to move img ${xPos},${yPos}`);
        await page.mouse.move(xPos, yPos,{steps:5});
        await this.delay(300)
        await page.mouse.down();
        await this.delay(200);
        //await page.mouse.move(destCx+1.5,yPos,{steps:5})
        while(xPos 

NOTES:

• 识别服务返回的缺口位置是相对与图片的坐标，需要转换成网页的坐标

  const destCx = cpcBox.x+puzz[0]+puzz[2]/2;
  

• 滑动滑块的时候需要微调每次滑动的距离

  xPos += getRandomInt(10,30);