#my function doesn't fully work as expected

6 messages · Page 1 of 1 (latest)

ornate maple
#
    println!("Downloading and encoding files...");
    let mut dl_count = 0u8;
    
    let mut sources: Vec<(String, Vec<(String, Vec<String>)>)> = vec![];
    for src in srcs_list {
        let mut filters: Vec<(String, Vec<String>)> = vec![];
        for list in src.lists {
            let mut domains: Vec<String> = vec![];
            for url in list.urls {
                let resp = match client.get(&url).send() {
                    Ok(ok) => ok,
                    Err(err) => {
                        println!("Error retrieving data from \"{url}\": {err}!\nSkipping...");
                        continue
                    }
                };
                
                if ! resp.status().is_success() {
                    println!("Error {}: Request for \"{url}\" was not successful!\nSkipping...", resp.status());
                    continue
                }
        
                let text: String = match resp.text() {
                    Ok(ok) => ok,
                    Err(err) => {
                        println!("Error encoding bytes to UTF-8: {err}!\nSkipping...");
                        continue
                    }
                };

                dl_count += 1;
#
                for line in text.lines() {
                    if line.is_empty() {
                        continue
                    }

                    if line.trim().starts_with('#') {
                        continue
                    }
                    
                    // Input file could be formatted like "hosts.txt"
                    let parts: Vec<&str> = line.split_whitespace().collect();
                    match parts.len() {
                        1 => domains.push(parts[0].to_string()),
                        2 => domains.push(parts[1].to_string()),
                        _ => continue
                    };
                }
            }
            filters.push((list.filter, domains));
        }
        sources.push((src.name, filters));
    }

    println!("{dl_count} files were successfully downloaded and encoded.");

    if sources.is_empty() {
        println!("No data was retrieved!");
        // ExitCode 74 is used to indicate an IO error
        return Ok(ExitCode::from(74))
    }

    println!("Creating backup directories...");

    let path_to_backup_string: String = match path_to_backup.to_str() {
        Some(path_to_backup_str) => path_to_backup_str.to_string(),
        None => {
            println!("Error: the \"path_to_backup\" is not unicode!");
            return Ok(ExitCode::from(65))
        }
    };

    let (year, month, day) = get_datetime();
    let new_dir_string = format!("{path_to_backup_string}/bkp/{year}/{month}/{day}/");
    let new_dir = Path::new(&new_dir_string);
    if ! new_dir.exists() {
        if let Err(err) = create_dir_all(new_dir) {
            println!("Error creating new directories: {new_dir_string}\nERR: {err}!");
            // ExitCode 73 is used to indicate that a file/directory could not be created
            return Ok(ExitCode::from(73))
        }
    }```
#
    println!("Querying Redis...");

    let mut found_count = 0u32;
    let mut cursor = 0u32;
    loop {
        let scan_keys: Vec<String>;
        (cursor, scan_keys) = scan(&mut connection, cursor, &"DBL:R:*".to_string())?;
        if cursor == 0 {
            break
        }

        if scan_keys.is_empty() {
            continue
        }

        let mut keys_set: HashSet<String> = HashSet::new();
        for key in scan_keys {
            keys_set.insert(key);
        }
        for (_source, filters) in &mut sources {
            for (filter, domains) in filters {
                let hkey_prefix = format!("DBL:R:{filter}:");
                domains.retain(|domain| {
                    let hkey = hkey_prefix.clone() + domain;
                    if keys_set.contains(&hkey) {
                        found_count += 1;
                        false
                    } else {
                        true
                    }
                })
            }
        }
    }

    println!("{found_count} rules were already found on Redis.");
    println!("Adding new rules to Redis and creating backup files...");
#

    let mut add_count = 0u32;
    let mut bkp_add_count = 0u32;
    for (source, filters) in sources {
        let path_string = format!("{new_dir_string}{source}");
        let bkp_path = Path::new(&path_string);
        let mut bkp_file: Option<File> = None;
        if let Ok(file) = fs::OpenOptions::new().append(true).create(true).open(bkp_path) {
            bkp_file = Some(file)
        }

        for (filter, domains) in filters {
            let hkey_prefix = format!("DBL:R:{filter}:");
            for domain in domains {
                let hkey = hkey_prefix.clone() + &domain;
                if let Ok(count) = exec(&mut connection, "hset", &vec![hkey.clone(),
                    "A".to_string(), "1".to_string(),
                    "AAAA".to_string(), "1".to_string(),
                    "enabled".to_string(), "1".to_string(),
                    "date".to_string(), format!("{year}{month}{day}"),
                    "source".to_string(), source.clone()]) {
                    if count != 0 {
                        add_count += 1;

                        if let Some(ref mut file) = bkp_file {
                            if let Ok(_) = writeln!(file, "{}", hkey) {
                                bkp_add_count += 1
                            }
                        }
                    }
                }
            }
        }
    }

    println!("{add_count} new rules were added to Redis.");
    println!("{bkp_add_count} new rules were backed up.");

    Ok(ExitCode::SUCCESS)
}```
#

this issue is that after "querying redis..." I am removing keys from my vecs when they are found on Redis.
when executing twice with the same input, the second time all the keys should be already found on Redis but instead a small percentage (always the same keys) are not found despite all being on the Redis db.