first version
This commit is contained in:
Marco "Capypara" Köpcke
commit
c7b9211c18
16 changed files with 3268 additions and 0 deletions
3
.gitignore
vendored
Normal file
3
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
/.envrc
|
||||
.idea
|
||||
target
|
||||
3
.nixignore
Normal file
3
.nixignore
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
/.envrc
|
||||
.idea
|
||||
target
|
||||
21
COPYING
Normal file
21
COPYING
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
MIT License
|
||||
|
||||
Copyright (c) 2024 Marco Köpcke
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
96
README.md
Normal file
96
README.md
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
Nix Jetbrains Plugins
|
||||
=====================
|
||||
|
||||
This repository contains derivations for ALL plugins from the Jetbrains Marketplace.
|
||||
|
||||
It is regularly updated to include all current plugins in their latest compatible version.
|
||||
|
||||
If any derivations fail to build or plugins are missing, please open an issue.
|
||||
We asume that plugins are not re-released with the same version number, so if a plugin does this for any reason,
|
||||
they might break and need manual fixing in this repository.
|
||||
|
||||
The plugins exported by this Flake are indexed by their IDE, version and then plugin ID.
|
||||
You can find the plugin IDs at the bottom of Marketplace pages.
|
||||
|
||||
The plugin list is only updated for IDEs from the current and previous year, for other IDEs the list may be stale.
|
||||
|
||||
## How to setup
|
||||
|
||||
### With Flakes
|
||||
|
||||
#### Inputs:
|
||||
|
||||
```nix
|
||||
inputs.nix-jebrains-plugins.url = "github:theCapypara/nix-jebrains-plugins";
|
||||
```
|
||||
|
||||
#### Usage:
|
||||
```nix
|
||||
let
|
||||
pluginList = [
|
||||
nix-jebrains-plugins.plugins."${system}".idea-ultimate."2024.3"."com.intellij.plugins.watcher"
|
||||
];
|
||||
in {
|
||||
# ... see "How to use"
|
||||
}
|
||||
```
|
||||
|
||||
### Without flakes
|
||||
|
||||
```nix
|
||||
let
|
||||
system = builtins.currentSystem;
|
||||
plugins =
|
||||
(import (builtins.fetchGit {
|
||||
url = "https://github.com/theCapypara/nix-jebrains-plugins";
|
||||
ref = "refs/heads/main";
|
||||
rev = "<latest commit hash>";
|
||||
})).plugins."${system}";
|
||||
pluginList = [
|
||||
plugins.idea-ultimate."2024.3"."com.intellij.plugins.watcher"
|
||||
];
|
||||
in {
|
||||
# ... see "How to use"
|
||||
}
|
||||
```
|
||||
|
||||
## How to use
|
||||
|
||||
The plugins can be used with ``jetbrains.plugins.addPlugins``:
|
||||
|
||||
```nix
|
||||
{
|
||||
environment.systemPackages = [
|
||||
# See "How to setup" for definition of `pluginList`.
|
||||
pkgs.jetbrains.plugins.addPlugins pkgs.jetbrains.idea-ultimate pluginList
|
||||
];
|
||||
}
|
||||
```
|
||||
|
||||
## Convenience functions (`lib`)
|
||||
The flake exports some convenience functions that can be used to make adding plugins to your IDEs
|
||||
easier.
|
||||
|
||||
These functions are only compatible and tested with the latest stable nixpkgs version.
|
||||
|
||||
### `buildIdeWithPlugins`
|
||||
|
||||
Using this function you can build an IDE using a set of named plugins from this Flake. The function
|
||||
will automatically figure out what IDE and version the plugin needs to be for.
|
||||
|
||||
#### Arguments:
|
||||
|
||||
1. `pkgs.jetbrains` from nixpkgs.
|
||||
2. The `pkgs.jetbrains` key of the IDE to build or download.
|
||||
3. A list of plugin IDs to install.
|
||||
|
||||
#### Example:
|
||||
|
||||
```nix
|
||||
{
|
||||
environment.systemPackages = with nix-jebrains-plugins.lib."${system}"; [
|
||||
# Adds the latest IDEA Ultimate version with the latest compatible version of "com.intellij.plugins.watcher".
|
||||
buildIdeWithPlugins pkgs.jetbrains "idea-ultimate" ["com.intellij.plugins.watcher"]
|
||||
];
|
||||
}
|
||||
```
|
||||
62
dev.nix
Normal file
62
dev.nix
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
{
|
||||
mkShell,
|
||||
lib,
|
||||
stdenv,
|
||||
|
||||
llvmPackages_latest,
|
||||
clang,
|
||||
rustup,
|
||||
pkg-config,
|
||||
|
||||
openssl,
|
||||
zlib,
|
||||
}:
|
||||
let
|
||||
overrides = (builtins.fromTOML (builtins.readFile ./generator/rust-toolchain.toml));
|
||||
extraLibs = [
|
||||
stdenv.cc.cc.lib
|
||||
zlib
|
||||
];
|
||||
libPath = lib.makeLibraryPath extraLibs;
|
||||
in
|
||||
mkShell {
|
||||
RUSTC_VERSION = overrides.toolchain.channel;
|
||||
# https://github.com/rust-lang/rust-bindgen#environment-variables
|
||||
LIBCLANG_PATH = lib.makeLibraryPath [ llvmPackages_latest.libclang.lib ];
|
||||
shellHook = ''
|
||||
export PATH=$PATH:''${CARGO_HOME:-~/.cargo}/bin
|
||||
export PATH=$PATH:''${RUSTUP_HOME:-~/.rustup}/toolchains/$RUSTC_VERSION-x86_64-unknown-linux-gnu/bin/
|
||||
'';
|
||||
# Add precompiled library to rustc search path
|
||||
RUSTFLAGS = (
|
||||
builtins.map (a: ''-L ${a}/lib'') [
|
||||
# add libraries here (e.g. pkgs.libvmi)
|
||||
]
|
||||
);
|
||||
LD_LIBRARY_PATH = libPath;
|
||||
# Add glibc, clang, glib, and other headers to bindgen search path
|
||||
BINDGEN_EXTRA_CLANG_ARGS =
|
||||
# Includes normal include path
|
||||
(builtins.map (a: ''-I"${a}/include"'') [
|
||||
# add dev libraries here (e.g. pkgs.libvmi.dev)
|
||||
])
|
||||
# Includes with special directory paths
|
||||
++ [
|
||||
''-I"${llvmPackages_latest.libclang.lib}/lib/clang/${llvmPackages_latest.libclang.version}/include"''
|
||||
];
|
||||
|
||||
nativeBuildInputs = [ ];
|
||||
|
||||
buildInputs =
|
||||
[
|
||||
openssl
|
||||
zlib
|
||||
pkg-config
|
||||
]
|
||||
## RUST
|
||||
++ [
|
||||
clang
|
||||
llvmPackages_latest.bintools
|
||||
rustup
|
||||
];
|
||||
}
|
||||
77
flake.lock
generated
Normal file
77
flake.lock
generated
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
{
|
||||
"nodes": {
|
||||
"flake-utils": {
|
||||
"inputs": {
|
||||
"systems": "systems"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1731533236,
|
||||
"narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1733412085,
|
||||
"narHash": "sha256-FillH0qdWDt/nlO6ED7h4cmN+G9uXwGjwmCnHs0QVYM=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "4dc2fc4e62dbf62b84132fe526356fbac7b03541",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"ref": "nixos-24.11",
|
||||
"repo": "nixpkgs",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"root": {
|
||||
"inputs": {
|
||||
"flake-utils": "flake-utils",
|
||||
"nixpkgs": "nixpkgs",
|
||||
"systems": "systems_2"
|
||||
}
|
||||
},
|
||||
"systems": {
|
||||
"locked": {
|
||||
"lastModified": 1681028828,
|
||||
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"systems_2": {
|
||||
"locked": {
|
||||
"lastModified": 1681028828,
|
||||
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"type": "github"
|
||||
}
|
||||
}
|
||||
},
|
||||
"root": "root",
|
||||
"version": 7
|
||||
}
|
||||
48
flake.nix
Normal file
48
flake.nix
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
{
|
||||
inputs = {
|
||||
nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11";
|
||||
flake-utils.url = "github:numtide/flake-utils";
|
||||
systems.url = "github:nix-systems/default";
|
||||
};
|
||||
|
||||
outputs =
|
||||
{
|
||||
self,
|
||||
nixpkgs,
|
||||
systems,
|
||||
flake-utils,
|
||||
}:
|
||||
flake-utils.lib.eachSystem (import systems) (
|
||||
system:
|
||||
let
|
||||
pkgs = import nixpkgs {
|
||||
inherit system;
|
||||
};
|
||||
in
|
||||
rec {
|
||||
plugins = pkgs.callPackage ./plugins.nix { };
|
||||
|
||||
packages = {
|
||||
_nix-jebrains-plugins-generator = pkgs.callPackage ./generator/pkg.nix { };
|
||||
};
|
||||
|
||||
devShells = {
|
||||
default = pkgs.callPackage ./dev.nix { };
|
||||
};
|
||||
|
||||
lib = {
|
||||
# Using this function you can build an IDE using a set of named plugins from this Flake. The function
|
||||
# will automatically figure out what IDE and version the plugin needs to be for.
|
||||
# See README.
|
||||
buildIdeWithPlugins =
|
||||
jetbrains: ide-name: plugin-ids:
|
||||
let
|
||||
ide = jetbrains."${ide-name}";
|
||||
in
|
||||
jetbrains.plugins.addPlugins ide (
|
||||
builtins.map (p: plugins."${ide.pname}"."${ide.version}"."${p}") plugin-ids
|
||||
);
|
||||
};
|
||||
}
|
||||
);
|
||||
}
|
||||
2142
generator/Cargo.lock
generated
Normal file
2142
generator/Cargo.lock
generated
Normal file
File diff suppressed because it is too large
Load diff
22
generator/Cargo.toml
Normal file
22
generator/Cargo.toml
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
[package]
|
||||
name = "nix-jebrains-plugins-generator"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
[dependencies]
|
||||
anyhow = "1"
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
reqwest = { version = "0.12", features = ["json", "stream"] }
|
||||
clap = { version = "4.5", features = ["derive"] }
|
||||
log = "0.4"
|
||||
log4rs = "1.2"
|
||||
serde = "1"
|
||||
serde-xml-rs = "0.6"
|
||||
serde_json = "1"
|
||||
futures = "0.3"
|
||||
tokio-retry2 = "0.5"
|
||||
nix-base32 = "0.2"
|
||||
base64 = "0.22"
|
||||
version-compare = "0.2"
|
||||
lazy_static = "1.5"
|
||||
which = "7"
|
||||
32
generator/pkg.nix
Normal file
32
generator/pkg.nix
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
{
|
||||
rustPlatform,
|
||||
cargo,
|
||||
rustc,
|
||||
pkg-config,
|
||||
openssl,
|
||||
}:
|
||||
rustPlatform.buildRustPackage {
|
||||
pname = "nix-jebrains-plugins-generator";
|
||||
version = "0.1.0";
|
||||
|
||||
src = ./.;
|
||||
|
||||
cargoLock = {
|
||||
lockFile = ./Cargo.lock;
|
||||
};
|
||||
|
||||
nativeBuildInputs = [
|
||||
pkg-config
|
||||
cargo
|
||||
rustc
|
||||
openssl
|
||||
];
|
||||
|
||||
buildInputs = [
|
||||
openssl
|
||||
];
|
||||
|
||||
meta = {
|
||||
mainProgram = "nix-jebrains-plugins-generator";
|
||||
};
|
||||
}
|
||||
2
generator/rust-toolchain.toml
Normal file
2
generator/rust-toolchain.toml
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
[toolchain]
|
||||
channel = "stable"
|
||||
176
generator/src/ides.rs
Normal file
176
generator/src/ides.rs
Normal file
|
|
@ -0,0 +1,176 @@
|
|||
use log::warn;
|
||||
use serde::Deserialize;
|
||||
use std::collections::HashSet;
|
||||
|
||||
const JETBRAINS_VERSIONS: &str = "https://www.jetbrains.com/updates/updates.xml";
|
||||
|
||||
const PROCESSED_VERSION_PREFIXES: &[&str] = &["2027.", "2026.", "2025.", "2024.", "2023."];
|
||||
|
||||
#[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)]
|
||||
pub enum IdeProduct {
|
||||
IntelliJUltimate,
|
||||
IntelliJCommunity,
|
||||
PhpStorm,
|
||||
WebStorm,
|
||||
PyCharmProfessional,
|
||||
PyCharmCommunity,
|
||||
RubyMine,
|
||||
CLion,
|
||||
GoLand,
|
||||
DataGrip,
|
||||
DataSpell,
|
||||
Rider,
|
||||
AndroidStudio,
|
||||
RustRover,
|
||||
Aqua,
|
||||
Writerside,
|
||||
Mps,
|
||||
}
|
||||
impl IdeProduct {
|
||||
fn try_from_code(code: &str) -> Option<Self> {
|
||||
Some(match code {
|
||||
"IU" => IdeProduct::IntelliJUltimate,
|
||||
"IC" => IdeProduct::IntelliJCommunity,
|
||||
"PS" => IdeProduct::PhpStorm,
|
||||
"WS" => IdeProduct::WebStorm,
|
||||
"PY" => IdeProduct::PyCharmProfessional,
|
||||
"PC" => IdeProduct::PyCharmCommunity,
|
||||
"RM" => IdeProduct::RubyMine,
|
||||
"CL" => IdeProduct::CLion,
|
||||
"GO" => IdeProduct::GoLand,
|
||||
"DB" => IdeProduct::DataGrip,
|
||||
"DS" => IdeProduct::DataSpell,
|
||||
"RD" => IdeProduct::Rider,
|
||||
"AI" => IdeProduct::AndroidStudio,
|
||||
"RR" => IdeProduct::RustRover,
|
||||
"QA" => IdeProduct::Aqua,
|
||||
"WRS" => IdeProduct::Writerside,
|
||||
"MPS" => IdeProduct::Mps,
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
||||
#[allow(unused)] // maybe useful later
|
||||
pub fn product_code(&self) -> &str {
|
||||
match self {
|
||||
IdeProduct::IntelliJUltimate => "IU",
|
||||
IdeProduct::IntelliJCommunity => "IC",
|
||||
IdeProduct::PhpStorm => "PS",
|
||||
IdeProduct::WebStorm => "WS",
|
||||
IdeProduct::PyCharmProfessional => "PY",
|
||||
IdeProduct::PyCharmCommunity => "PC",
|
||||
IdeProduct::RubyMine => "RM",
|
||||
IdeProduct::CLion => "CL",
|
||||
IdeProduct::GoLand => "GO",
|
||||
IdeProduct::DataGrip => "DB",
|
||||
IdeProduct::DataSpell => "DS",
|
||||
IdeProduct::Rider => "RD",
|
||||
IdeProduct::AndroidStudio => "AI",
|
||||
IdeProduct::RustRover => "RR",
|
||||
IdeProduct::Aqua => "QA",
|
||||
IdeProduct::Writerside => "WRS",
|
||||
IdeProduct::Mps => "MPS",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn nix_key(&self) -> &str {
|
||||
match self {
|
||||
IdeProduct::IntelliJUltimate => "idea-ultimate",
|
||||
IdeProduct::IntelliJCommunity => "idea-community",
|
||||
IdeProduct::PhpStorm => "phpstorm",
|
||||
IdeProduct::WebStorm => "webstorm",
|
||||
IdeProduct::PyCharmProfessional => "pycharm-professional",
|
||||
IdeProduct::PyCharmCommunity => "pycharm-community",
|
||||
IdeProduct::RubyMine => "ruby-mine",
|
||||
IdeProduct::CLion => "clion",
|
||||
IdeProduct::GoLand => "goland",
|
||||
IdeProduct::DataGrip => "datagrip",
|
||||
IdeProduct::DataSpell => "dataspell",
|
||||
IdeProduct::Rider => "rider",
|
||||
IdeProduct::AndroidStudio => "android-studio",
|
||||
IdeProduct::RustRover => "rust-rover",
|
||||
IdeProduct::Aqua => "aqua",
|
||||
IdeProduct::Writerside => "writerside",
|
||||
IdeProduct::Mps => "MPS",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq, Clone, Hash)]
|
||||
pub struct IdeVersion {
|
||||
pub ide: IdeProduct,
|
||||
pub version: String,
|
||||
pub build_number: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Deserialize)]
|
||||
pub struct Products {
|
||||
product: Vec<Product>,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Deserialize)]
|
||||
pub struct Product {
|
||||
code: Vec<String>,
|
||||
channel: Option<Vec<Channel>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Deserialize, Clone)]
|
||||
pub struct Channel {
|
||||
build: Vec<Build>,
|
||||
id: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Deserialize, Clone)]
|
||||
pub struct Build {
|
||||
number: String,
|
||||
version: String,
|
||||
}
|
||||
|
||||
pub async fn collect_ids() -> anyhow::Result<Vec<IdeVersion>> {
|
||||
let products: Products =
|
||||
serde_xml_rs::from_str(&reqwest::get(JETBRAINS_VERSIONS).await?.text().await?)?;
|
||||
|
||||
let mut already_processed = HashSet::new();
|
||||
let mut versions: Vec<IdeVersion> = Vec::new();
|
||||
|
||||
for product in products.product {
|
||||
for code in product.code {
|
||||
if let Some(ideobj) = IdeProduct::try_from_code(&code) {
|
||||
if already_processed.insert(ideobj) {
|
||||
if let Some(channels) = product.channel.as_ref() {
|
||||
for channel in channels {
|
||||
if channel.id.ends_with("RELEASE-licensing-RELEASE") {
|
||||
for build in &channel.build {
|
||||
if allowed_build_version(&build.version) {
|
||||
versions.push(IdeVersion {
|
||||
ide: ideobj,
|
||||
version: build.version.clone(),
|
||||
build_number: build.number.clone(),
|
||||
})
|
||||
} else {
|
||||
warn!(
|
||||
"Ignoring {} {}: too old",
|
||||
ideobj.nix_key(),
|
||||
build.version
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(versions)
|
||||
}
|
||||
|
||||
fn allowed_build_version(version: &str) -> bool {
|
||||
for allowed in PROCESSED_VERSION_PREFIXES {
|
||||
if version.starts_with(allowed) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
21
generator/src/logging.rs
Normal file
21
generator/src/logging.rs
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
use log::LevelFilter;
|
||||
use log4rs::append::console::{ConsoleAppender, Target};
|
||||
use log4rs::config::{Appender, Root};
|
||||
use log4rs::{init_config, Config, Handle};
|
||||
|
||||
pub fn setup_logging() -> anyhow::Result<Handle> {
|
||||
let threshold = if cfg!(debug_assertions) {
|
||||
LevelFilter::Debug
|
||||
} else {
|
||||
LevelFilter::Info
|
||||
};
|
||||
|
||||
let config = Config::builder()
|
||||
.appender(Appender::builder().build(
|
||||
"stderr",
|
||||
Box::new(ConsoleAppender::builder().target(Target::Stderr).build()),
|
||||
))
|
||||
.build(Root::builder().appender("stderr").build(threshold))?;
|
||||
|
||||
Ok(init_config(config)?)
|
||||
}
|
||||
48
generator/src/main.rs
Normal file
48
generator/src/main.rs
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
mod ides;
|
||||
mod logging;
|
||||
mod plugins;
|
||||
|
||||
use clap::Parser;
|
||||
use log::info;
|
||||
use std::path::PathBuf;
|
||||
use tokio::try_join;
|
||||
|
||||
#[derive(Parser)]
|
||||
struct Cli {
|
||||
#[arg(short, long)]
|
||||
output_path: PathBuf,
|
||||
}
|
||||
const PLUGIN_INDICES: &[&str] = &[
|
||||
"https://downloads.marketplace.jetbrains.com/files/pluginsXMLIds.json",
|
||||
"https://downloads.marketplace.jetbrains.com/files/jbPluginsXMLIds.json",
|
||||
];
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
let cli = Cli::parse();
|
||||
_ = logging::setup_logging();
|
||||
info!("Starting...");
|
||||
|
||||
let (ides, mut plugins, jb_plugins) = try_join!(
|
||||
ides::collect_ids(),
|
||||
plugins::index(PLUGIN_INDICES[0]),
|
||||
plugins::index(PLUGIN_INDICES[1])
|
||||
)?;
|
||||
|
||||
info!(
|
||||
"Indexing {} IDE versions, {} plugins and {} Jetbrains plugins.",
|
||||
ides.len(),
|
||||
plugins.len(),
|
||||
jb_plugins.len()
|
||||
);
|
||||
plugins.extend_from_slice(&jb_plugins);
|
||||
|
||||
info!("Loading old database.");
|
||||
let cache_db = plugins::db_cache_load(&cli.output_path).await?;
|
||||
info!("Beginning plugin download...");
|
||||
let db = plugins::build_db(&cache_db, &ides, &plugins).await?;
|
||||
info!("Saving DB...");
|
||||
plugins::db_save(&cli.output_path, db).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
419
generator/src/plugins.rs
Normal file
419
generator/src/plugins.rs
Normal file
|
|
@ -0,0 +1,419 @@
|
|||
use crate::ides::IdeVersion;
|
||||
use anyhow::anyhow;
|
||||
use base64::prelude::BASE64_STANDARD;
|
||||
use base64::Engine;
|
||||
use futures::stream::iter;
|
||||
use futures::{StreamExt, TryStreamExt};
|
||||
use lazy_static::lazy_static;
|
||||
use log::{debug, info, warn};
|
||||
use reqwest::{Client, StatusCode};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::borrow::Cow;
|
||||
use std::collections::{BTreeMap, HashMap, HashSet};
|
||||
use std::fs::exists;
|
||||
use std::future;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Stdio;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::fs::{read_to_string, write};
|
||||
use tokio::process::Command;
|
||||
use tokio::sync::RwLock;
|
||||
use tokio::time::timeout;
|
||||
use tokio_retry2::strategy::ExponentialBackoff;
|
||||
use tokio_retry2::{Retry, RetryError};
|
||||
use version_compare::Version;
|
||||
use which::which;
|
||||
|
||||
const ALL_PLUGINS_JSON: &str = "all_plugins.json";
|
||||
|
||||
lazy_static! {
|
||||
static ref NIX_PREFETCH_URL: PathBuf =
|
||||
which("nix-prefetch-url").expect("nix-prefetch-url not in PATH");
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Serialize, PartialOrd, PartialEq, Ord, Eq, Hash)]
|
||||
pub struct PluginVersion(String);
|
||||
|
||||
impl PluginVersion {
|
||||
const SEPARATOR: &'static str = "/--/";
|
||||
pub fn new(name: &str, version: &str) -> Self {
|
||||
Self(format!("{}{}{}", name, Self::SEPARATOR, version))
|
||||
}
|
||||
}
|
||||
|
||||
type PluginCache = HashMap<PluginVersion, PluginDbEntry>;
|
||||
// Plugins for which download requests have 404ed
|
||||
type FourOFourCache = HashSet<PluginVersion>;
|
||||
|
||||
pub struct PluginDb {
|
||||
// all_plugins caches all entries, ides contains references to them.
|
||||
all_plugins: BTreeMap<PluginVersion, &'static PluginDbEntry>,
|
||||
ides: HashMap<IdeVersion, BTreeMap<String, String>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Deserialize)]
|
||||
pub struct PluginDetails {
|
||||
category: Option<PluginDetailsCategory>,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Deserialize)]
|
||||
pub struct PluginDetailsCategory {
|
||||
#[serde(rename = "idea-plugin")]
|
||||
idea_plugin: Vec<PluginDetailsIdeaPlugin>,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Deserialize)]
|
||||
pub struct PluginDetailsIdeaPlugin {
|
||||
version: String,
|
||||
#[serde(rename = "idea-version")]
|
||||
idea_version: PluginDetailsIdeaVersion,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Deserialize)]
|
||||
pub struct PluginDetailsIdeaVersion {
|
||||
#[serde(rename = "since-build")]
|
||||
since_build: Option<String>,
|
||||
#[serde(rename = "until-build")]
|
||||
until_build: Option<String>,
|
||||
}
|
||||
|
||||
impl PluginDb {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
all_plugins: Default::default(),
|
||||
ides: Default::default(),
|
||||
}
|
||||
}
|
||||
pub fn insert(
|
||||
&mut self,
|
||||
ideversion: &IdeVersion,
|
||||
name: &str,
|
||||
version: &str,
|
||||
entry: &PluginDbEntry,
|
||||
) {
|
||||
let version_entry = self.ides.entry(ideversion.clone()).or_default();
|
||||
// We leak here since self-referential structs are otherwise a nightmare and it doesn't
|
||||
// really matter in this CLI app.
|
||||
self.all_plugins
|
||||
.entry(PluginVersion::new(name, version))
|
||||
.or_insert_with(|| Box::leak(Box::new(entry.clone())));
|
||||
version_entry.insert(name.to_string(), version.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize)]
|
||||
pub struct PluginDbEntry {
|
||||
#[serde(rename = "p")]
|
||||
pub path: String,
|
||||
#[serde(rename = "h")]
|
||||
pub hash: String,
|
||||
}
|
||||
|
||||
pub async fn index(url: &str) -> anyhow::Result<Vec<String>> {
|
||||
Ok(reqwest::get(url).await?.json().await?)
|
||||
}
|
||||
|
||||
pub async fn db_cache_load(out_dir: &Path) -> anyhow::Result<PluginCache> {
|
||||
let file = out_dir.join(ALL_PLUGINS_JSON);
|
||||
if exists(&file)? {
|
||||
Ok(serde_json::from_str(&read_to_string(file).await?)?)
|
||||
} else {
|
||||
Ok(PluginCache::new())
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn build_db(
|
||||
cache: &PluginCache,
|
||||
ides: &[IdeVersion],
|
||||
pluginkeys: &[String],
|
||||
) -> anyhow::Result<PluginDb> {
|
||||
let cache = Arc::new(cache);
|
||||
let client = Arc::new(
|
||||
Client::builder()
|
||||
.timeout(Duration::from_secs(600))
|
||||
.build()?,
|
||||
);
|
||||
let fof_cache = Arc::new(RwLock::new(FourOFourCache::new()));
|
||||
let db = Arc::new(RwLock::new(PluginDb::new()));
|
||||
|
||||
let mut futures = Vec::new();
|
||||
|
||||
for pluginkey in pluginkeys {
|
||||
let fof_cache = fof_cache.clone();
|
||||
let db = db.clone();
|
||||
let client = client.clone();
|
||||
let cache = cache.clone();
|
||||
|
||||
// Create a future that will be retried 3 times, has a timeout of 1200 seconds per try
|
||||
// and polls process_plugin to process this plugin for this IDE version. process_plugin
|
||||
// will update the database.
|
||||
futures.push(async move {
|
||||
Retry::spawn(ExponentialBackoff::from_millis(250).take(3), move || {
|
||||
let fof_cache = fof_cache.clone();
|
||||
let db = db.clone();
|
||||
let client = client.clone();
|
||||
let cache = cache.clone();
|
||||
async move {
|
||||
let res = timeout(
|
||||
Duration::from_secs(1200),
|
||||
process_plugin(
|
||||
db.clone(),
|
||||
client.clone(),
|
||||
ides,
|
||||
pluginkey,
|
||||
&cache,
|
||||
fof_cache.clone(),
|
||||
),
|
||||
)
|
||||
.await;
|
||||
match res {
|
||||
Ok(Ok(v)) => Ok(v),
|
||||
Ok(Err(e)) => {
|
||||
warn!("failed plugin processing {pluginkey}: {e}. Might retry.");
|
||||
Err(RetryError::transient(e))
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(
|
||||
"failed plugin processing {pluginkey} due to timeout. Might retry."
|
||||
);
|
||||
Err(RetryError::transient(anyhow!("timeout").context(e)))
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
});
|
||||
}
|
||||
|
||||
iter(futures)
|
||||
.buffered(16)
|
||||
// TODO: try_collect does not exit early. try_all does. Is there any better way to do this?
|
||||
.try_all(|()| future::ready(true))
|
||||
.await?;
|
||||
|
||||
Ok(Arc::into_inner(db).unwrap().into_inner())
|
||||
}
|
||||
|
||||
/// Various hacks to support (or skip) some very odd cases
|
||||
fn hacks_for_details_key(pluginkey: &str) -> Option<&str> {
|
||||
match pluginkey {
|
||||
// The former is the real ID, but it trips up the plugin endpoint...
|
||||
"23.bytecode-disassembler" => Some("bytecode-disassembler"),
|
||||
// Has invalid version numbers
|
||||
"com.valord577.mybatis-navigator" => None,
|
||||
// ZIP contains invalid file names
|
||||
"io.github.kings1990.FastRequest" => None,
|
||||
v => Some(v),
|
||||
}
|
||||
}
|
||||
|
||||
async fn process_plugin(
|
||||
db: Arc<RwLock<PluginDb>>,
|
||||
client: Arc<Client>,
|
||||
ides: &[IdeVersion],
|
||||
pluginkey: &str,
|
||||
cache: &PluginCache,
|
||||
fof_cache: Arc<RwLock<FourOFourCache>>,
|
||||
) -> anyhow::Result<()> {
|
||||
debug!("Processing {pluginkey}...");
|
||||
|
||||
let Some(pluginkey_for_details) = hacks_for_details_key(pluginkey) else {
|
||||
warn!("{pluginkey}: plugin is marked as broken, skipping...");
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
let req = client
|
||||
.get(format!(
|
||||
"https://plugins.jetbrains.com/plugins/list?pluginId={}",
|
||||
pluginkey_for_details
|
||||
))
|
||||
.send()
|
||||
.await?;
|
||||
if !req.status().is_success() {
|
||||
return Err(anyhow!(
|
||||
"{} failed details request: {}",
|
||||
pluginkey,
|
||||
req.status()
|
||||
));
|
||||
}
|
||||
let details: PluginDetails = serde_xml_rs::from_str(&req.text().await?)?;
|
||||
|
||||
let Some(category) = details.category else {
|
||||
warn!("{pluginkey}: No plugin details available. Skipping!");
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
let mut versions = category.idea_plugin;
|
||||
versions.sort_by(|a, b| b.version.cmp(&a.version));
|
||||
|
||||
for ide in ides {
|
||||
match supported_version(ide, &versions) {
|
||||
None => warn!("{pluginkey}: IDE {ide:?} not supported."),
|
||||
Some(version) => {
|
||||
let entry =
|
||||
get_db_entry(&client, pluginkey, &version.version, &db, cache, &fof_cache)
|
||||
.await?;
|
||||
if let Some(entry) = entry {
|
||||
let mut lck = db.write().await;
|
||||
lck.insert(ide, pluginkey, &version.version, &entry);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn supported_version<'a>(
|
||||
ide: &IdeVersion,
|
||||
versions: &'a Vec<PluginDetailsIdeaPlugin>,
|
||||
) -> Option<&'a PluginDetailsIdeaPlugin> {
|
||||
let build_version = Version::from(&ide.build_number).unwrap();
|
||||
for version in versions {
|
||||
if let Some(min) = version.idea_version.since_build.as_ref() {
|
||||
if build_version < Version::from(&min.replace(".*", ".0")).unwrap() {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if let Some(max) = version.idea_version.until_build.as_ref() {
|
||||
if build_version > Version::from(&max.replace(".*", ".99999999")).unwrap() {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
return Some(version);
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
async fn get_db_entry<'a>(
|
||||
client: &Client,
|
||||
pluginkey: &str,
|
||||
version: &str,
|
||||
current_db: &RwLock<PluginDb>,
|
||||
cache: &'a PluginCache,
|
||||
fof_cache: &RwLock<FourOFourCache>,
|
||||
) -> anyhow::Result<Option<Cow<'a, PluginDbEntry>>> {
|
||||
let key = PluginVersion::new(pluginkey, version);
|
||||
// Look in current_db
|
||||
{
|
||||
let db_lck = current_db.read().await;
|
||||
let v = db_lck.all_plugins.get(&key);
|
||||
if let Some(v) = v {
|
||||
return Ok(Some(Cow::Borrowed(v)));
|
||||
}
|
||||
};
|
||||
// Look in cache
|
||||
if let Some(v) = cache.get(&key) {
|
||||
return Ok(Some(Cow::Borrowed(v)));
|
||||
}
|
||||
|
||||
{
|
||||
if fof_cache.read().await.contains(&key) {
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
|
||||
info!(
|
||||
"{}@{}: Plugin not yet cached, downloading for hash...",
|
||||
pluginkey, version
|
||||
);
|
||||
|
||||
let req = client
|
||||
.head(format!(
|
||||
"https://plugins.jetbrains.com/plugin/download?pluginId={}&version={}",
|
||||
pluginkey, version
|
||||
))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
if req.status() == StatusCode::NOT_FOUND {
|
||||
warn!("{}@{}: not available: skipping", pluginkey, version);
|
||||
fof_cache.write().await.insert(key);
|
||||
return Ok(None);
|
||||
} else if !req.status().is_success() {
|
||||
return Err(anyhow!(
|
||||
"{}@{}: failed download HEAD request: {}",
|
||||
pluginkey,
|
||||
version,
|
||||
req.status()
|
||||
));
|
||||
}
|
||||
|
||||
const PREFIX_OF_ALL_URLS: &str = "https://downloads.marketplace.jetbrains.com/";
|
||||
// Query parameters don't seem to result in different files, probably only for analytics.
|
||||
// Remove them to save some space.
|
||||
// Also remove the https://downloads.marketplace.jetbrains.com/ prefix.
|
||||
let mut url = req.url().clone();
|
||||
url.set_query(None);
|
||||
let url = url.to_string();
|
||||
|
||||
let is_jar = url.ends_with(".jar");
|
||||
let hash_nix32 = get_nix32_hash(
|
||||
&format!("{pluginkey}-{version}-source").replace(|c: char| !c.is_alphanumeric(), "-"),
|
||||
&url,
|
||||
!is_jar,
|
||||
is_jar,
|
||||
)
|
||||
.await?;
|
||||
let hash = BASE64_STANDARD.encode(
|
||||
nix_base32::from_nix_base32(&hash_nix32)
|
||||
.ok_or_else(|| anyhow!("{}@{}: failed decoding nix hash", pluginkey, version,))?,
|
||||
);
|
||||
|
||||
let path = url
|
||||
.strip_prefix(PREFIX_OF_ALL_URLS)
|
||||
.expect("expect all URLs to start with prefix.")
|
||||
.to_string();
|
||||
|
||||
Ok(Some(Cow::Owned(PluginDbEntry { path, hash })))
|
||||
}
|
||||
|
||||
async fn get_nix32_hash(
|
||||
name: &str,
|
||||
url: &str,
|
||||
unpack: bool,
|
||||
executable: bool,
|
||||
) -> anyhow::Result<String> {
|
||||
let mut parameters = Vec::with_capacity(8);
|
||||
parameters.push("--type");
|
||||
parameters.push("sha256");
|
||||
parameters.push("--name");
|
||||
parameters.push(name);
|
||||
if unpack {
|
||||
parameters.push("--unpack");
|
||||
}
|
||||
if executable {
|
||||
parameters.push("--executable");
|
||||
}
|
||||
parameters.push(url);
|
||||
|
||||
let child = Command::new(&*NIX_PREFETCH_URL)
|
||||
.args(parameters)
|
||||
.stdout(Stdio::piped())
|
||||
.kill_on_drop(true)
|
||||
.spawn()?;
|
||||
|
||||
let result = child.wait_with_output().await?;
|
||||
if !result.status.success() {
|
||||
return Err(anyhow!("nix-prefetch-url failed for {url}"));
|
||||
}
|
||||
let out = String::from_utf8(result.stdout)?.trim().to_string();
|
||||
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
pub async fn db_save(output_folder: &Path, db: PluginDb) -> anyhow::Result<()> {
|
||||
// all plugins
|
||||
let out_path = output_folder.join(ALL_PLUGINS_JSON);
|
||||
debug!("Generating {out_path:?}...");
|
||||
write(out_path, serde_json::to_string_pretty(&db.all_plugins)?).await?;
|
||||
|
||||
// mappings
|
||||
let output_folder = output_folder.join("ides");
|
||||
for (ide, plugins) in db.ides {
|
||||
let out_path = output_folder.join(format!("{}-{}.json", ide.ide.nix_key(), ide.version));
|
||||
debug!("Generating {out_path:?}...");
|
||||
write(out_path, serde_json::to_string_pretty(&plugins)?).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
96
plugins.nix
Normal file
96
plugins.nix
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
{
|
||||
lib,
|
||||
fetchurl,
|
||||
fetchzip,
|
||||
stdenv,
|
||||
}:
|
||||
with builtins;
|
||||
with lib;
|
||||
let
|
||||
SEPARATOR = "/--/";
|
||||
|
||||
fetchPluginSrc =
|
||||
{ url, hash }:
|
||||
let
|
||||
isJar = hasSuffix ".jar" url;
|
||||
fetcher = if isJar then fetchurl else fetchzip;
|
||||
in
|
||||
fetcher {
|
||||
executable = isJar;
|
||||
inherit url hash;
|
||||
};
|
||||
|
||||
downloadPlugin =
|
||||
{
|
||||
name,
|
||||
version,
|
||||
url,
|
||||
hash,
|
||||
}:
|
||||
let
|
||||
isJar = hasSuffix ".jar" url;
|
||||
installPhase =
|
||||
if isJar then
|
||||
''
|
||||
runHook preInstall
|
||||
mkdir -p $out && cp $src $out
|
||||
runHook postInstall
|
||||
''
|
||||
else
|
||||
''
|
||||
runHook preInstall
|
||||
mkdir -p $out && cp -r . $out
|
||||
runHook postInstall
|
||||
'';
|
||||
in
|
||||
stdenv.mkDerivation {
|
||||
inherit name version;
|
||||
src = fetchPluginSrc { inherit url hash; };
|
||||
dontUnpack = isJar;
|
||||
inherit installPhase;
|
||||
};
|
||||
|
||||
readGeneratedDir = attrNames (
|
||||
filterAttrs (name: _: hasSuffix ".json" name) (readDir ./generated/ides)
|
||||
);
|
||||
|
||||
# Folds into the set of { IDENAME = { VERSION = [ x y ]; }; }
|
||||
buildIdeVersionMap = (
|
||||
accu: value:
|
||||
accu
|
||||
// {
|
||||
"${value.version}" = (accu."${value.version}" or { }) // value.value;
|
||||
}
|
||||
);
|
||||
|
||||
# Find and construct plugin from a list of plugins
|
||||
findPlugin =
|
||||
pluginList: name: version:
|
||||
let
|
||||
key = "${name}${SEPARATOR}${version}";
|
||||
match = pluginList."${key}";
|
||||
in
|
||||
{
|
||||
inherit name version;
|
||||
url = "https://downloads.marketplace.jetbrains.com/${match.p}";
|
||||
hash = "sha256-${match.h}";
|
||||
};
|
||||
|
||||
allPlugins = fromJSON (readFile ./generated/all_plugins.json);
|
||||
in
|
||||
(groupBy' buildIdeVersionMap { } (x: x.ideName) (
|
||||
map (
|
||||
jsonFile:
|
||||
let
|
||||
# Split the JSON filename into IDENAME-VERSION and remove json suffix
|
||||
parts = splitString "-" (removeSuffix ".json" jsonFile);
|
||||
in
|
||||
{
|
||||
ideName = concatStrings (intersperse "-" (init parts));
|
||||
version = elemAt parts ((length parts) - 1);
|
||||
value = mapAttrs (k: v: downloadPlugin (findPlugin allPlugins k v)) (
|
||||
fromJSON (readFile (./generated/ides + "/${jsonFile}"))
|
||||
);
|
||||
}
|
||||
) readGeneratedDir
|
||||
))
|
||||
Loading…
Add table
Add a link
Reference in a new issue